File size: 7,861 Bytes
cd6775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | from openai import OpenAI
import json
import os
import string
import regex
import time
from collections import Counter
import joblib
from tqdm import tqdm
import pandas as pd
import pandas as pd
import socket
from flask import Flask, request, jsonify
app = Flask(__name__)
HOST = '127.0.0.1'
PORT = 50007
client = OpenAI(
api_key=os.environ.get("DEEPSEEK_API_KEY", ""),
base_url='https://api.deepseek.com/v1',
)
import re
def extract_between_asterisks(text):
# 使用正则表达式匹配 `**` 之间的内容
pattern = r'\*\*(.*?)\*\*'
matches = re.findall(pattern, text)
return matches
def eval_em(p_ls, g_ls):
assert len(p_ls) == len(g_ls)
cnt = 0
for idx in range(len(p_ls)):
pred = p_ls[idx]
gold = g_ls[idx]
if gold in pred:
cnt += 1
return cnt/len(p_ls)
def excute(questions, answers):
ans_ls = []
pred_ls= []
query_ls = []
acc_ls = []
acc = 0
cnt = 0
for idx in range(len(questions)):
print(cnt)
q = questions[idx]
answer = answers[idx]
query_ls.append(q)
ans_ls.append(answer)
feedback_tmp_ls = []
round_count = 0
message_keys_list = [{"role": "user", "content":
"""Construct a global reasoning chain for this complex [Question] : " {} " You should generate a query to the search engine based on
what you already know at each step of the reasoning chain, starting with [Query].
If you know the answer for [Query], generate it starting with [Answer].
You can try to generate the final answer for the [Question] by referring to the [Query]-[Answer] pairs, starting with [Final
Content]. The final answer for the [Question] can only be yes or no, marked with **.
If you don't know the answer, generate a query to search engine based on what you already know and do not know, starting with
[Unsolved Query].
For example:
[Question]: "Where do greyhound buses that are in the birthplace of Spirit If...'s performer leave from? "
[Query 1]: Who is the performer of Spirit If... ?
If you don't know the answer:
[Unsolved Query]: Who is the performer of Spirit If... ?
If you know the answer:
[Answer 1]: The performer of Spirit If... is Kevin Drew.
[Query 2]: Where was Kevin Drew born?
If you don't know the answer:
[Unsolved Query]: Where was Kevin Drew born?
If you know the answer:
[Answer 2]: Toronto.
[Query 3]: Where do greyhound buses in Toronto leave from?
If you don't know the answer:
[Unsolved Query]: Where do greyhound buses in Toronto leave from?
If you know the answer:
[Answer 3]: Toronto Coach Terminal.
[Final Content]: The performer of Spirit If... is Kevin Drew [1]. Kevin Drew was born in Toronto [2]. Greyhound buses in
Toronto leave from Toronto
Coach Terminal [3]. So the final answer is Toronto Coach Terminal.
[Question]:"Which magazine was started first Arthur’s Magazine or First for Women?"
[Query 1]: When was Arthur’s Magazine started?
[Answer 1]: 1844.
[Query 2]: When was First for Women started?
[Answer 2]: 1989
[Final Content]: Arthur’s Magazine started in 1844 [1]. First for Women started in 1989 [2]. So Arthur’s Magazine was started
first. So the answer is Arthur’s Magazi
[Question]: {}
""".format(q,q)}]
feedback_answer = 'continue'
predict_answer = ''
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
while round_count < 5 and not feedback_answer == 'end':
print('round is {}'.format(round_count))
try:
rsp = client.chat.completions.create(
model='deepseek-chat',
messages=message_keys_list
)
round_count += 1
input_str = rsp.choices[0].message.content
message_keys_list.append({"role": "assistant", "content": input_str})
print('solving......')
predict_answer += input_str
sock.send(input_str.encode())
print('send message {}'.format(input_str))
feedback = sock.recv(10240).decode()
print('feedback is '+feedback)
if feedback == 'end':
break
#[Query]:xxxx<SEP>[Answer]:xxxx<SEP>[Reference]:xxxx<SEP>
feedback_list = feedback.split('<SEP>')
if not 'Unsolved Query' in feedback:
new_prompt = """
According to this Reference, the answer for "{}" should be "{}",
you can change your answer based on the Reference and continue constructing the reasoning chain to give the final answer for [Question]:{}
Reference: {}
""".format(feedback_list[0],feedback_list[1],q,feedback_list[2])
else:
new_prompt = """
According to this Reference, the answer for "{}" should be "{}",
you can give your answer based on the Reference and continue constructing the reasoning chain to give the final answer for [Question]：{}
Reference: {}
""".format(feedback_list[0],feedback_list[1],q,feedback_list[2])
message_keys_list.append({"role": "user", "content":new_prompt})
feedback_tmp_ls.append((feedback_list[0], feedback_list[2]))
except:
print('start_idx is {}'.format(k))
sock.send('end'.encode())
sock.close()
return k
if not feedback_answer == 'end':
sock.send('end'.encode())
sock.close()
print(message_keys_list)
last_assistant = None
for entry in reversed(message_keys_list):
if entry['role'] == 'assistant':
last_assistant = entry
break
pred_ans = last_assistant['content'].split('[Final Content]')[-1].lower()
pred_ls.append(pred_ans.split('.')[-2])
# if answer is True:
# answer = 'yes'
# else:
# answer = 'no'
if answer in extract_between_asterisks(pred_ans):
acc += 1
acc_ls.append(1)
else:
acc_ls.append(0)
cnt += 1
return jsonify({
"query_ls": query_ls,
"ans_ls": ans_ls,
"acc_ls": acc_ls
})
@app.route('/execute', methods=["GET"])
def api_search():
if request.method == "GET":
queries = request.args.getlist("query")
answers = request.args.getlist("answers")
return excute(queries, answers)
else:
return '', 405
if __name__ == '__main__':
app.run(host='0.0.0.0', port=50009)
|