一种图文多模态RAG
作者： 知岩智隧 来源： 知岩智隧
RAG的优点不必多说了，笔者曾构建了一个基于图文检索的围岩图像“以图搜图”应用，基本功能是：输入图像、检索TopK图文(image-text pairs)。尽管该应用中数据库里对检索的文本字段做了整理，检索图像时能够在历史数据中快速找到相关的图文描述，从而为决策提出工一定的参考。笔者使用的milvus向量数据库构建的图文数据存储，将图像经过embedding处理，并保存了相应的图像描述信息，效果可参考以下视频。
以图搜图的"围岩地质信息检索"
为了进一步增强该应用的功能，比如能够围绕检索的结果做一些总结描述等功能，参考了纯文本RAG的技术流程，笔者设计了一个针对图文数据的RAG检索增强生成方案，如下图所示。这个方案中模型方面涉及：embedding model,Image captioning model,LLM-chat/agent，这个方案使用了LLava对输入图像描述的生成模型，使用gemma模型进行最终的总结生成。LLM模型全部使用OLLAMA Serve模式，通过ollama-python调用API进行相应的任务处理，具体参考文末代码。相比使用双向量数据库方案（可参考llama-index的示例：https://docs.llamaindex.ai/en/stable/examples/multi_modal/ollama_cookbook/），本方案仅使用了单个向量数据库，采用了两个小规模的LLM(LLava:7b(~4GB)+gemma:2b(~2GB))进行协作来完成问答和围绕知识库的问答。
mRAG
直接上示例效果：
输入图像为（图像md5:9e4bdf9fde930967169f8dd5b85448d1）：
RAG回答：
隧道地质图文mRAG
具体技术细节，欢迎交流！以下为方案的完整实现代码，不到100行！
from routers.TMRL.feature_extractor_ResNet50_onnx import FeatureExtractorResNet50
from pymilvus import Collection,connections
from pymilvus import Role, utility
import base64
import json
from ollama import generate
from ollama import chat
import requests
COLLECTION_NAME="RockImageQAHNSW" #milvus collection
HOST="xx.xx.xx.xx"
PORT="8083"
connections.connect(host=HOST, port=PORT,user='xxxx',password='xxxxxxxxxx')
def readImageBase64(image_path):
with open(image_path,"rb") as f:
return base64.b64encode(f.read()).decode()
def getSimilarImageDecriptions(image_path,TopK=3,similarFilter=0.3):
############################ get embedding for search ##############################################
FE=FeatureExtractorResNet50()#use ResNet50 for image embedding
query_vec=FE.extract_feature(image_path,False)
query_vec=query_vec.tolist()
############################### Start Search #######################################################
search_params = {"metric_type": "L2", "params": {"nprobe": 200}, "offset": 1,"M": 64,"efConstruction":256,"ef":30}
collection = Collection(COLLECTION_NAME) # Get an existing collection.
collection.load()
search_results = collection.search(
data=[query_vec],
anns_field="embedding", #Name of the field to search on.
param=search_params,
limit=TopK,
expr=None,
round_decimal=3,
consistency_level="Strong",
)
query_result=[]
for id in search_results[0].ids:
res=collection.query(
expr="data_id =="+str(id),
offset=0,
limit=1,
output_fields = ["image_path","description"],
consistency_level="Strong"
)
res[0]['desc_kw']=json.loads(res[0]['desc_kw'])
query_result.append(res[0])
similar_image_decriptions=[]
for i in range(len(query_result)):
if search_results[0].distances[i]>=similarFilter:#相似性过滤
similar_image_decriptions.append(query_result[i]["description"])
#similar_image_decriptions.append({"id":search_results[0].ids[i],"distance":search_results[0].distances[i],"info":query_result[i]})
return similar_image_decriptions
def getImageDescription(image_path):
input_imagedata=readImageBase64(image_path)#see ollama-python
vllm_response=generate('llava:13b', "answer in Chinese, describe this image's content and check is it rock image or has rock parts:", images=[input_imagedata], stream=False)
if vllm_response:
image_description=vllm_response["response"]
else:
image_description="给定图片无描述，请自行推理"
return image_description
def QAtemplate_EN(input_text,image_description,similar_image_descriptions,has_image=True):
if has_image:
image_num=len(similar_image_descriptions)
if image_num==0:
context=f"just answer the question as you can based on desciption:\'{image_description}\',question:\'{input_text}\'."
else:
context=f"the given image has desciption:\'{image_description}\',and it has {image_num} similar images with descriptions,similarty in descending order:\n"
for id,sim in enumerate(similar_image_descriptions):
context+=f"similar image {id}:\'{sim}\'\n"
context+=f"please reasoning the given image charactristics and answer the question :\"{input_text}\" based the given image's description and its similar images' descriptions.just do as you can."
else:
context=f"just answer the question as you can,question:\'{input_text}\'."
return context
def QAtemplate_ZH(input_text, image_description, similar_image_descriptions, has_image=True):
if has_image:
image_num = len(similar_image_descriptions)
if image_num==0:
context=f"请根据给定图片的描述:\'{image_description}\',回答问题：\'{input_text}\'"
else:
context = f"给定图片的描述：\'{image_description}\'，它有 {image_num} 张相似的图片，描述如下：\n"
for id, sim in enumerate(similar_image_descriptions):
context += f"相似图片 {id}: \'{sim}\'\n"
context += f"请根据输入的图片描述及其相似图片的描述推理输入图片的特征，并回答问题：\"{input_text}\"，尽力而为。"
else:
context = f"请根据问题：\'{input_text}\'，尽力给出你的回答。"#可以使用纯文本知识库回答
return context
TopK=3
image_path="9e4bdf9fde930967169f8dd5b85448d1.jpg" #读取图片数据，输入可能为：纯图像、纯文本、图像+文本
input_text="请描述图片中岩石的特征"#"please describe the image"
similar_image_decriptions=getSimilarImageDecriptions(image_path,TopK)
# if has image #use mLLM(llava) for image description and check image type
image_description=getImageDescription(image_path)
#############################create context with similar images descriptions######################
context=QAtemplate_ZH(input_text,image_description,similar_image_decriptions,True)
print(context)
############################ final answer the question with context ##############################
messages = [{'role': 'user','content': context}]
response = chat('gemma:2b', messages=messages)#from ollama,gemma:2b,gemma:7b and other as you want
print("*"*10,"answer","*"*10)
print(response['message']['content'])
以上，欢迎交流！
更多AI工具，参考Github-AiBard123，国内AiBard123
