| """Recompute the published evidence results from the provided rows and raw outputs.""" |
| import argparse,json,sys |
| from pathlib import Path |
|
|
| def main(folder): |
| sys.path.insert(0,str(folder/'reproduce')) |
| from common import read,grade,summary,compare,digest |
| report=read(folder/'evaluation.json');assert digest(folder/'benchmark/rows.json')==report['rows_sha256'] |
| rows={r['id']:r for r in read(folder/'benchmark/rows.json') if r['split']=='test'} |
| sources={'candidate':'hf_candidate.json','hf_base':'hf_base.json','native_base':'native_base.jsonl','native_challenger':'native_challenger.jsonl','gpt_oss_20b':'gpt_oss_20b.json','cpu_trained':'cpu_trained.json'};results={} |
| for name,file in sources.items(): |
| path=folder/'benchmark'/file;raw=[json.loads(s) for s in path.read_text(encoding='utf-8').splitlines()] if file.endswith('.jsonl') else read(path);raw=[r for r in raw if r['split']=='test'] |
| assert {r['id'] for r in raw}==set(rows) and len(raw)==len(rows) |
| results[name]=[] |
| for r in raw: |
| g=grade('evidence',rows[r['id']],r['raw']);g['passed']=g['passed'] and r.get('clean_stop',True) |
| assert g['passed']==r['passed'] |
| results[name].append(dict(id=r['id'],group_id=rows[r['id']]['group_id'],**g)) |
| assert summary(results[name])['accuracy']==report['conditions'][name]['accuracy'] |
| results['lexical_rules']=[dict(id=r['id'],group_id=r['group_id'],**grade('evidence',r,r['rule_output'])) for r in rows.values()] |
| comparisons={n:compare(results['candidate'],rs) for n,rs in results.items() if n!='candidate'} |
| assert all(c['interval95'][0]>0 for c in comparisons.values()) |
| print(json.dumps(dict(verified=True,n=len(rows),conditions={n:summary(rs) for n,rs in results.items()},comparisons=comparisons),indent=2)) |
|
|
| if __name__=='__main__': |
| parser=argparse.ArgumentParser();parser.add_argument('folder',nargs='?',type=Path,default=Path(__file__).resolve().parent);main(parser.parse_args().folder) |
|
|
