File size: 2,426 Bytes
2183ffa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """Additional live appended-question, cold parallel, and input-bound checks."""
import asyncio,json,time,secrets,argparse
from pathlib import Path
import httpx
from benchmark_material import fixture,picks
async def main(out):
out.mkdir(parents=True,exist_ok=False)
async with httpx.AsyncClient(base_url='http://127.0.0.1:8243',timeout=180) as c:
async def post(path,body):
t=time.perf_counter();r=await c.post(path,json=body);r.raise_for_status();return r.json(),(time.perf_counter()-t)*1000
rows=[]
for repeat in range(3):
for beta in [False,True]:
case=fixture('beta' if beta else 'alpha',80,beta=beta)
p,pm=await post('/v1/materials',{'state':case['state']});mid=p['material_id']
r,ms=await post(f'/v1/materials/{mid}/questions',{'questions':case['questions'],'concurrency':4})
assert r['metrics']['all_candidates_reused_entire_material']
rows.append({'case':case['name'],'repeat':repeat,'mode':'cold-parallel','preparation':p,'result':r,'wall_ms':pm+ms,
'correct':sum(picks(r)[k]==v for k,v in case['expected'].items())})
extra={'pens':{'instructions':'How many pens, excluding notebooks, are in Kestrel?','options':['5','7','9']},
'receipt':{'instructions':'Does the material explicitly establish that Mina has a receipt?','options':['Yes','No']},
'visitor':{'instructions':'What is the name of the registered contractor arriving without an escort?','options':['Mina','Lin','Kestrel']}}
expected={'pens':3 if beta else 2,'receipt':1,'visitor':2}
r,ms=await post(f'/v1/materials/{mid}/questions',{'questions':extra,'concurrency':4})
assert picks(r)==expected and r['metrics']['all_candidates_reused_entire_material']
rows.append({'case':case['name'],'repeat':repeat,'mode':'appended-new-questions','result':r,'wall_ms':ms,'correct':3})
# Rejected calls must not remove a valid material handle.
bad=await c.post(f'/v1/materials/{mid}/questions',json={'questions':{'q':{'instructions':'token '*8000,'options':['a','b']}}});assert bad.status_code==422
deleted=await c.delete('/v1/materials/'+mid);assert deleted.status_code==200
(out/'results.json').write_text(json.dumps(rows,indent=2));print([(r['case'],r['mode'],round(r['wall_ms']),r['correct']) for r in rows],flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--out',type=Path,required=True);asyncio.run(main(p.parse_args().out))
|