Hugging Face's logo

Veblen34
/
Query-decompose-baselines

Safetensors
English
query-decomposition
retrieval
rag
baselines
Model card Files
xet
 Community
Query-decompose-baselines
File size: 2,528 Bytes
cd6775d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
from transformers import pipeline
import torch
import pandas as pd
import sys
import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', 'document_retrieval', 'Decompose_retrieval'))
from pipline import *
import openai
import logging
# 禁用 OpenAI 客户端的 INFO 级别日志
logging.getLogger("openai").setLevel(logging.ERROR)




def ask_gpt(query):
    messages = [{"role": "user", "content": query}]
    result = client.chat.completions.create(messages=messages, model="meta-llama/Meta-Llama-3-8B-Instruct")
    return result.choices[0].message.content

def get_accuracy_re(true_imgs, re_imgs):
    # print(true_imgs)
    # print(re_imgs)
    cnt = 0
    for i in range(len(true_imgs)):
        i_gt = true_imgs[i].rsplit('.', 1)[0]
        image_retrieval = re_imgs[i].rsplit('.', 1)[0]
        if i_gt == image_retrieval:
            cnt += 1
    return cnt/len(true_imgs)



def get_rag_answers(q_list, sub_q_list, dataset_name = "multiqa"):
    if dataset_name == "multiqa":
        ans_img_list, re_im = eval_acc(q_list, sub_q_list, patch_emb_by_img_ls)
        ans = []
        re_imgs = []
        for i in range(len(ans_img_list)):
            ans.append(ans_img_list[i][0])
            re_imgs.append(re_im[i])
        return ans, re_imgs
    


def exact_match(predictions, ground_truths):
    # sum = 0
    score = []
    for i in range(len(predictions)):
        pred = predictions[i].lower()
        gt = ground_truths[i].lower()
        if gt == pred:
            score.append(1)
        else:
            score.append(0)
    return score


def get_accuracy_multiqa(predictions, ground_truths):
    score = exact_match(predictions, ground_truths)
    return score


dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "multiqa_test.csv")
client = OpenAI(api_key="0",base_url="http://0.0.0.0:8000/v1")

single_task_df = pd.read_csv(dataset_path, header=0)
raw_data = single_task_df[['query', 'answer', 'image']]
raw_queries = list(raw_data['query'])
true_answers = list(raw_data['answer'])
true_images = list(raw_data['image'])


 
raw_op_prompts = []
for i in range(len(raw_data)):
    raw_pred = ask_gpt(raw_queries[i])
    raw_op_prompts.append(raw_pred)


pred_answers, re_imgs = get_rag_answers(raw_queries, raw_op_prompts, "multiqa")


accuracy = get_accuracy_multiqa(
  predictions = pred_answers,
  ground_truths = true_answers
)
re_acc = get_accuracy_re(true_images , re_imgs)

accuracy = np.average(accuracy)


print(accuracy, re_acc)