File size: 1,651 Bytes
0690c04 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | """Compare both exploratory prompts with the pinned original BF16 runtime."""
import argparse,json,sys,time
from pathlib import Path
import numpy as np
import torch
from duplex_cases import CASES
from evaluate_duplex import VARIANTS,state_text,summary
from d3_format import render
from d3_runtime import D3
p=argparse.ArgumentParser();p.add_argument('--source',required=True);p.add_argument('--native-results',required=True);p.add_argument('--output',required=True);a=p.parse_args()
torch.set_num_threads(4);ref=D3.from_pretrained(a.source,device='cpu',verify='full');actual=json.loads(Path(a.native_results).read_text());result={}
for name,q in VARIANTS.items():
rows=[]
for c,n in zip(CASES,actual[name]['records'],strict=True):
assert n['id']==c['id']
text=render(ref.tokenizer,state_text(c),q,ref.codes)
seq=ref.tokenizer.encode(text,add_special_tokens=False)
t=time.perf_counter();prob=ref.probabilities([seq],[3])[0]
keys=list(q['criteria']);r=dict(n,reference_probabilities=prob,reference_choice=keys[int(np.argmax(prob))],max_probability_error=float(abs(np.array(prob)-np.array(n['probabilities'])).max()),reference_seconds=time.perf_counter()-t)
r['choice_matches']=r['choice']==r['reference_choice'];rows.append(r)
result[name]=dict(records=rows,choices_matched=sum(r['choice_matches'] for r in rows),cases=len(rows),max_probability_error=max(r['max_probability_error'] for r in rows),reference_business_correct=sum(r['reference_choice']==r['expected'] for r in rows))
Path(a.output).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');print(name,{k:v for k,v in result[name].items() if k!='records'},flush=True)
|