| """Additional live appended-question, cold parallel, and input-bound checks.""" |
| import asyncio,json,time,secrets,argparse |
| from pathlib import Path |
| import httpx |
| from benchmark_material import fixture,picks |
|
|
| async def main(out): |
| out.mkdir(parents=True,exist_ok=False) |
| async with httpx.AsyncClient(base_url='http://127.0.0.1:8243',timeout=180) as c: |
| async def post(path,body): |
| t=time.perf_counter();r=await c.post(path,json=body);r.raise_for_status();return r.json(),(time.perf_counter()-t)*1000 |
| rows=[] |
| for repeat in range(3): |
| for beta in [False,True]: |
| case=fixture('beta' if beta else 'alpha',80,beta=beta) |
| p,pm=await post('/v1/materials',{'state':case['state']});mid=p['material_id'] |
| r,ms=await post(f'/v1/materials/{mid}/questions',{'questions':case['questions'],'concurrency':4}) |
| assert r['metrics']['all_candidates_reused_entire_material'] |
| rows.append({'case':case['name'],'repeat':repeat,'mode':'cold-parallel','preparation':p,'result':r,'wall_ms':pm+ms, |
| 'correct':sum(picks(r)[k]==v for k,v in case['expected'].items())}) |
| extra={'pens':{'instructions':'How many pens, excluding notebooks, are in Kestrel?','options':['5','7','9']}, |
| 'receipt':{'instructions':'Does the material explicitly establish that Mina has a receipt?','options':['Yes','No']}, |
| 'visitor':{'instructions':'What is the name of the registered contractor arriving without an escort?','options':['Mina','Lin','Kestrel']}} |
| expected={'pens':3 if beta else 2,'receipt':1,'visitor':2} |
| r,ms=await post(f'/v1/materials/{mid}/questions',{'questions':extra,'concurrency':4}) |
| assert picks(r)==expected and r['metrics']['all_candidates_reused_entire_material'] |
| rows.append({'case':case['name'],'repeat':repeat,'mode':'appended-new-questions','result':r,'wall_ms':ms,'correct':3}) |
| |
| bad=await c.post(f'/v1/materials/{mid}/questions',json={'questions':{'q':{'instructions':'token '*8000,'options':['a','b']}}});assert bad.status_code==422 |
| deleted=await c.delete('/v1/materials/'+mid);assert deleted.status_code==200 |
| (out/'results.json').write_text(json.dumps(rows,indent=2));print([(r['case'],r['mode'],round(r['wall_ms']),r['correct']) for r in rows],flush=True) |
| if __name__=='__main__': |
| p=argparse.ArgumentParser();p.add_argument('--out',type=Path,required=True);asyncio.run(main(p.parse_args().out)) |
|
|
