| """Compare both exploratory prompts with the pinned original BF16 runtime.""" |
| import argparse,json,sys,time |
| from pathlib import Path |
| import numpy as np |
| import torch |
| from duplex_cases import CASES |
| from evaluate_duplex import VARIANTS,state_text,summary |
| from d3_format import render |
| from d3_runtime import D3 |
| p=argparse.ArgumentParser();p.add_argument('--source',required=True);p.add_argument('--native-results',required=True);p.add_argument('--output',required=True);a=p.parse_args() |
| torch.set_num_threads(4);ref=D3.from_pretrained(a.source,device='cpu',verify='full');actual=json.loads(Path(a.native_results).read_text());result={} |
| for name,q in VARIANTS.items(): |
| rows=[] |
| for c,n in zip(CASES,actual[name]['records'],strict=True): |
| assert n['id']==c['id'] |
| text=render(ref.tokenizer,state_text(c),q,ref.codes) |
| seq=ref.tokenizer.encode(text,add_special_tokens=False) |
| t=time.perf_counter();prob=ref.probabilities([seq],[3])[0] |
| keys=list(q['criteria']);r=dict(n,reference_probabilities=prob,reference_choice=keys[int(np.argmax(prob))],max_probability_error=float(abs(np.array(prob)-np.array(n['probabilities'])).max()),reference_seconds=time.perf_counter()-t) |
| r['choice_matches']=r['choice']==r['reference_choice'];rows.append(r) |
| result[name]=dict(records=rows,choices_matched=sum(r['choice_matches'] for r in rows),cases=len(rows),max_probability_error=max(r['max_probability_error'] for r in rows),reference_business_correct=sum(r['reference_choice']==r['expected'] for r in rows)) |
| Path(a.output).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');print(name,{k:v for k,v in result[name].items() if k!='records'},flush=True) |
|
|
