File size: 1,403 Bytes
1831ecd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | """Full batch runner: actual BFCL runtime, scorer, cache and per-task evidence."""
import json
from common import read_json
from helpers import row,state
def test_batch_runner_executes_four_categories_then_reuses_verified_tasks(monkeypatch,tmp_path,config):
from evaluation import runner
from modeling import client
from benchmarks.scoring import canonical_answer
rows=[row(10,c) for c in config['bfcl_only']['categories']]
answers={r['messages'][-1]['content']:canonical_answer(r) for r in rows}
calls=[]
def chat(messages,**kwargs):
calls.append(messages[-1]['content'])
return {'role':'assistant','content':json.dumps(answers[messages[-1]['content']])},{'completion_tokens':5},'fixture'
monkeypatch.setattr(client,'chat',chat)
monkeypatch.setattr(client,'ensure_served',lambda *args:None)
monkeypatch.setattr(client,'verify_serving_profile',lambda *args:{'fixture':True})
current=state(tmp_path);output=tmp_path/'evaluation'
result=runner.evaluate(rows,current,config,output)
assert result['complete'] and result['score']==1. and result['resolved']==4
assert len(calls)==4 and result['output_tokens']==20
for outcome in result['results']:
assert read_json(__import__('pathlib').Path(outcome['episode_dir'])/'static-generation.json')
assert runner.evaluate(rows,current,config,output)==result and len(calls)==4
|