File size: 1,887 Bytes
0019be1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | from tqdm import tqdm
import json
import re
dataset_file = "./llm_for_code/results/09-07-mistral_iffp_lora4_0_10000.json"
allpairs = []
with open(dataset_file, 'r') as file:
for l in file:
allpairs.append(json.loads(l))
TP = 0
TN = 0
FP = 0
FN = 0
try:
responses = []
#allpairs = allpairs[:2]
for data in tqdm(allpairs):
pair = data['progs']
#prompt = file_content + pair['prog1']['scode'] + "\nProgram 2:"
#prompt += pair['prog2']['scode']+ "\n### Response:"
result=data['response']
end_ind = result.find("###")
res = result
if end_ind>0:
res = result[:end_ind]
#print(result)
#print("$$$$")
#print(res)
ans = pair['prog1']['probid'] == pair['prog2']['probid']
correct = False
if ans:
if "Yes" in res or "yes" in res:
TP += 1
correct = True
else: #"No" in res or "no" in res:
FN += 1
#print("My Yes")
if ans == False:
if "No" in res or "no" in res:
TN += 1
correct = True
else:
FP += 1
#print("My No")
responses.append({
'progs' : pair,
'response': res,
'extracted_answer': correct
})
with open(output_file, 'w+') as outfile:
for response in responses:
outfile.write(f"{json.dumps(response)}\n")
print(f"TPR: {TP/(TP+FN)}\nFPR: {TN/(TN+FP)} \nAcc: {(TN+TP)/(TP+FP+TN+FN)} ")
except Exception as e:
print(f"TPR: {TP/(TP+FN)}\nFPR: {TN/(TN+FP)} \nAcc: {(TN+TP)/(TP+FP+TN+FN)} ")
print(f"TPR: {TP/(TP+FN)}\nFPR: {TN/(TN+FP)} \nAcc: {(TN+TP)/(TP+FP+TN+FN)} ")
|