File size: 2,437 Bytes
cd6775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | from transformers import pipeline
import torch
import pandas as pd
from transformers import AutoTokenizer
import requests
from openai import OpenAI
instructions = [
"As a query optimization expert, systematically evaluate the provided query Q, identify ambiguities, inefficiencies, and potential improvements, and refine it to ensure maximum clarity, precision, and alignment with the user's intent, while maintaining semantic accuracy and contextual coherence in the output.",
"As a query optimization expert, meticulously evaluate the provided query Q, identify and resolve ambiguities or inefficiencies, and refine it to achieve exceptional clarity, precision, and alignment with the user's intent, ensuring the output is concise, semantically robust, contextually accurate, and optimized for maximum effectiveness, readability, and relevance, while preserving the original meaning and enhancing overall utility."
]
dataset_path = '/home/icml01/multi_rag/RAG/opro-para/data/strategyqa/strategyqa_dev.csv'
raw_data = pd.read_csv(dataset_path)
raw_queries = list(raw_data['question'])
true_answers = list(raw_data['answer'])
true_answers = ['yes' if row else 'no' for row in true_answers]
acc_dic = {}
def send_request(FLASK_URL, questions, answers):
payload = {
"questions": questions,
"answers": answers
}
url = "http://127.0.0.1:50003/execute"
params = {
"query": questions,
"answers": answers
}
# 发送请求
response = requests.get(url, params=params, timeout=None)
response_data = response.json()
query_ls = response_data.get("query_ls", [])
ans_ls = response_data.get("ans_ls", [])
acc_ls = response_data.get("acc_ls", [])
return query_ls, ans_ls, acc_ls
def get_acc(instruction, queries, answers):
url = "http://127.0.0.1:8894/api/update_instruction"
new_instruction = {"instruction": instruction}
requests.post(url, json=new_instruction)
FLASK_URL = "http://127.0.0.1:50003/execute"
query_ls, ans_ls, acc_ls = send_request(FLASK_URL, raw_queries, true_answers)
average = sum(acc_ls) / len(acc_ls)
return average
cnt = 0
for instruction in instructions:
print(f"========= evaluating round: {cnt} ==========")
acc = get_acc(instruction, raw_queries, true_answers)
print(f"INS: {instruction}, accuracy: {acc}")
acc_dic[instruction] = acc
cnt += 1 |