evaluate.py

"""Evaluate supplied labeled decisions; no bundled public benchmark text."""
import argparse,json,statistics,time
from collections import defaultdict
from pathlib import Path
from bonsai_runtime.backend import Client
from bonsai_runtime.decision import DecisionEngine
p=argparse.ArgumentParser();p.add_argument('--input',required=True);p.add_argument('--output',required=True)
p.add_argument('--base',default='http://127.0.0.1:5991');p.add_argument('--profile',choices=['base','current'],default='base');p.add_argument('--adapter-id',type=int,default=0)
a=p.parse_args();engine=DecisionEngine(Client(a.base,None if a.profile=='base' else a.adapter_id));rows=[]
with Path(a.output).open('x') as out:
 for line in Path(a.input).read_text().splitlines():
  item=json.loads(line);request=item['request']
  for order in [0,1]:
   r=dict(request);options=r['options'];r['options']=options[order:]+options[:order]
   result=engine.decide(r);row=dict(id=item['id'],order=order,correct=result['answer']==item['expected'],result=result)
   out.write(json.dumps(row)+'\n');out.flush();rows.append(row)
groups=defaultdict(list)
for r in rows:groups[r['id']].append(r)
canonical=[r for r in rows if r['order']==0]
print(json.dumps(dict(profile=a.profile,requests=len(groups),canonical_correct=sum(r['correct'] for r in canonical),all_order_correct=sum(all(r['correct'] for r in rs) for rs in groups.values()),mean_seconds=statistics.mean(r['result']['seconds'] for r in rows)),indent=2))