| """Full batch runner: actual BFCL runtime, scorer, cache and per-task evidence.""" |
| import json |
| from common import read_json |
| from helpers import row,state |
|
|
|
|
| def test_batch_runner_executes_four_categories_then_reuses_verified_tasks(monkeypatch,tmp_path,config): |
| from evaluation import runner |
| from modeling import client |
| from benchmarks.scoring import canonical_answer |
| rows=[row(10,c) for c in config['bfcl_only']['categories']] |
| answers={r['messages'][-1]['content']:canonical_answer(r) for r in rows} |
| calls=[] |
| def chat(messages,**kwargs): |
| calls.append(messages[-1]['content']) |
| return {'role':'assistant','content':json.dumps(answers[messages[-1]['content']])},{'completion_tokens':5},'fixture' |
| monkeypatch.setattr(client,'chat',chat) |
| monkeypatch.setattr(client,'ensure_served',lambda *args:None) |
| monkeypatch.setattr(client,'verify_serving_profile',lambda *args:{'fixture':True}) |
| current=state(tmp_path);output=tmp_path/'evaluation' |
| result=runner.evaluate(rows,current,config,output) |
| assert result['complete'] and result['score']==1. and result['resolved']==4 |
| assert len(calls)==4 and result['output_tokens']==20 |
| for outcome in result['results']: |
| assert read_json(__import__('pathlib').Path(outcome['episode_dir'])/'static-generation.json') |
| assert runner.evaluate(rows,current,config,output)==result and len(calls)==4 |
|
|
