File size: 1,331 Bytes
cd6775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | import os
from transformers import pipeline
import torch
import pandas as pd
from transformers import AutoTokenizer
import requests
from openai import OpenAI
dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "strategyqa_dev.csv")
raw_data = pd.read_csv(dataset_path)
raw_queries = list(raw_data['question'])
true_answers = list(raw_data['answer'])
true_answers = ['yes' if row else 'no' for row in true_answers]
acc_dic = {}
def send_request(FLASK_URL, questions, answers):
payload = {
"questions": questions,
"answers": answers
}
url = "http://127.0.0.1:50008/execute"
params = {
"query": questions,
"answers": answers
}
# 发送请求
response = requests.get(url, params=params, timeout=None)
response_data = response.json()
query_ls = response_data.get("query_ls", [])
ans_ls = response_data.get("ans_ls", [])
acc_ls = response_data.get("acc_ls", [])
return query_ls, ans_ls, acc_ls
def get_acc(queries, answers):
FLASK_URL = "http://127.0.0.1:50008/execute"
query_ls, ans_ls, acc_ls = send_request(FLASK_URL, raw_queries, true_answers)
average = sum(acc_ls) / len(acc_ls)
return average
acc = get_acc( raw_queries, true_answers)
print(f"baseline, accuracy: {acc}")
|