"""Eight-category integration: real simulators/checkers, deterministic model fixtures. No model service calls; the optimizer test uses actual CPU autograd. It does not claim performance or GPU compatibility of the production Qwen weights. """ import copy import json import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace import pytest from common import digest, read_json, read_jsonl, write_json, file_sha256 from benchmarks import bfcl_multiturn as mt from benchmarks.bfcl_partition import CATEGORIES, split from benchmarks.formats import import_source from benchmarks.runtime import run_episode from benchmarks.bfcl_tasks import replay_samples, validate_sample from modeling import client from helpers import state, row as static_row ROOT=Path(__file__).resolve().parents[1] def multi_row(category='multi_turn_base', role='improve', index=0): raw={'id':category+'_'+str(index),'question':[[{'role':'user','content':'Add 1 and 2.'}], [{'role':'user','content':'Multiply the previous result by 2.'}]],'initial_config':{},'involved_classes':['MathAPI']} truth=[['add(a=1,b=2)'],['multiply(a=3,b=2)']] if category=='multi_turn_miss_param': raw['question'][0][0]['content']='Add 1 to a value I will give you next.' raw['question'][1][0]['content']='The value is 2.';truth=[[],['add(a=1,b=2)']] if category=='multi_turn_miss_func': raw['question'].insert(1,[]);raw['missed_function']={'1':['add']} truth=[[],['add(a=1,b=2)'],['multiply(a=3,b=2)']] raw['question'][0][0]['content']='Scenario '+category+' '+str(index)+'. '+raw['question'][0][0]['content'] r=mt.normalize(raw,category,truth) r['tools']=[t for t in r['tools'] if t['function']['name'] in ('add','multiply')] r.update(split_role=role,split_group=digest(['family',index])) return r def answer_from_trace(monkeypatch,row,extra='',backend='both'): trace=mt.replay_trace(row,extra);answers=iter(trace['steps']);seen=[] def answer(messages,**kwargs): step=next(answers);assert messages==step['messages'] seen.append(copy.deepcopy(messages)) return copy.deepcopy(step['response']),{'completion_tokens':3},digest(messages) if backend in ('both','student'):monkeypatch.setattr(client,'chat',answer) if backend in ('both','teacher'):monkeypatch.setattr(client,'data_chat',answer) return seen @pytest.fixture(scope='module') def imported(): spec=read_json(ROOT/'plan/bfcl-only/datasets.json')['sources'][0] for e in spec['files']: for k in ('path','answer_path'):e[k]=str(ROOT/e[k]) return import_source(spec)[0] def test_all_799_multiturn_references_reexecute_and_validate(imported): rows=[r for r in imported if mt.is_multiturn(r)] assert len(rows)==799 and len(imported)==1784 for r in rows: trace=mt.replay_trace(r) assert mt.verify_trace(r,trace)['valid'],r['original_id'] from benchmarks.scoring import canonical_answer,matches assert matches(r,canonical_answer(r)),r['original_id'] assert {r['category'] for r in imported}==set(CATEGORIES) def test_quarantined_upstream_row_has_an_explicit_tool_schedule_conflict(): raw=[json.loads(l) for l in (ROOT/'data/source/bfcl-multi_turn_miss_func-questions.jsonl').read_text(encoding='utf-8').splitlines()] raw=next(r for r in raw if r['id']=='multi_turn_miss_func_49') truth=[json.loads(l) for l in (ROOT/'data/source/bfcl-multi_turn_miss_func-answers.jsonl').read_text(encoding='utf-8').splitlines()] truth=next(r['ground_truth'] for r in truth if r['id']==raw['id']) r=mt.normalize(raw,'multi_turn_miss_func',truth) assert 'tail' not in {t['function']['name'] for t in mt.tools_at(r,1)} assert any(c.startswith('tail(') for c in truth[1]) with pytest.raises(ValueError,match='Reference trajectory'):mt.replay_trace(r) def test_split_keeps_all_scenario_variants_together(imported): spec=read_json(ROOT/'plan/bfcl-only/datasets.json');cfg=read_json(ROOT/'plan/bfcl-only/full.json') parts=split(imported,spec['role_minimum_groups_per_category'],cfg['seed']) assert {k:len(v) for k,v in parts.items()}=={'improve':692,'selection':321,'meta_eval':129,'audit':321,'sealed_test':321} families={} for role,rows in parts.items(): assert {r['category'] for r in rows}==set(CATEGORIES) for r in rows: if mt.is_multiturn(r):families.setdefault(r['explicit_group'],set()).add((role,r['split_group'])) assert len(families)==200 and all(len(roles)==1 for roles in families.values()) assert parts==split(imported,spec['role_minimum_groups_per_category'],cfg['seed']) @pytest.mark.parametrize('category',mt.CATEGORIES) def test_runtime_is_stateful_turn_aligned_and_no_future_inputs(monkeypatch,tmp_path,config,category): row=multi_row(category);initial=state(tmp_path);seen=answer_from_trace(monkeypatch,row) result=run_episode(row,initial,config,tmp_path/'episode') assert result['resolved'] and result['output_tokens']==len(seen)*3 trace=read_json(tmp_path/'episode/multi-turn-generation.json') assert mt.verify_trace(row,trace)['valid'] assert row['turns'][-1] not in seen[0] assert 'initial_config' not in json.dumps(seen) and 'expected_calls' not in json.dumps(seen) if category=='multi_turn_miss_func': assert '"name": "add"' not in seen[0][0]['content'] assert any('"name": "add"' in m[0]['content'] for m in seen[1:]) def test_simulators_are_isolated_and_cannot_execute_host_code(tmp_path): row=multi_row();env=mt.Environment(row) assert 'Error during execution' in env.execute([{'name':'__import__','arguments':{'name':'os'}}],set(env.methods))[0] assert 'Error during execution' in env.execute([{'name':'_load_scenario','arguments':{}}],set(env.methods))[0] for text in ['__import__("os").system("echo unsafe")','add(a=open("x"),b=2)','add(**{})']: with pytest.raises((ValueError,SyntaxError)):mt.parse_reference(text) def execute(_): r=multi_row('multi_turn_long_context');return mt.verify_trace(r,mt.replay_trace(r)) with ThreadPoolExecutor(max_workers=4) as pool:assert all(r['valid'] for r in pool.map(execute,range(12))) def test_parameter_mutation_does_not_modify_recorded_actions(imported): r=next(r for r in imported if r['original_id']=='multi_turn_base_90') trace=mt.replay_trace(r);before=copy.deepcopy(trace) assert mt.verify_trace(r,trace)['valid'] and trace==before def test_full_harness_schema_repair_is_replayed(monkeypatch,tmp_path,config): from harness import manager row=multi_row();initial=state(tmp_path);artifact=manager.initial_artifact() artifact['verification'].update(schema_check=True,max_repairs=1) initial['harness']=str(manager.save_version(tmp_path/'repair-harness',artifact)) trace=mt.replay_trace(row);messages=[{'role':'assistant','content':'[{"name":"add","arguments":{}}]'}] messages += [s['response'] for s in trace['steps']] queue=iter(messages) monkeypatch.setattr(client,'chat',lambda *a,**k:(next(queue),{'completion_tokens':2},'repair')) result=run_episode(row,initial,config,tmp_path/'episode') assert result['resolved'] actual=read_json(tmp_path/'episode/multi-turn-generation.json') assert len(actual['steps'][0]['repairs'])==1 and mt.verify_trace(row,actual)['valid'] @pytest.mark.parametrize('source',['student_retry','temporary_harness','teacher']) def test_data_correction_uses_one_bayesian_action_for_many_targets(monkeypatch,tmp_path,config,source): from controller import decision_router from benchmarks import bfcl_solver from harness import manager row=multi_row();initial=state(tmp_path) monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) failed=run_episode(row,initial,config,tmp_path/'failed');assert not failed['resolved'] original=decision_router.choose def choose(*a,**kw):kw['legal_actions']=[source];return original(*a,**kw) monkeypatch.setattr(decision_router,'choose',choose) extra='' if source=='temporary_harness': artifact=manager.initial_artifact();extra='Use actual tool returns in the next step.';artifact['instructions_extra']=extra guided={**initial,'harness':str(manager.save_version(tmp_path/'guided',artifact))} monkeypatch.setattr(bfcl_solver,'temporary_harness',lambda *a,**k:{'valid':True,'candidate':guided,'output_tokens':0}) seen=answer_from_trace(monkeypatch,row,extra) args=(initial,row,tmp_path/'failed',failed,config,tmp_path/'correct',tmp_path/'run',[]) result=bfcl_solver.correct(*args) assert result['complete'] and len(result['samples'])==4 and len(result['attempts'])==1 for s in result['samples']:assert validate_sample(s,{('bfcl',row['task_id']):row}) saved=read_json(tmp_path/'correct/attempt-1/result.json') assert saved['measurement']['bernoulli_observations']==1 assert saved['measurement']['target_episode_count']==1 assert bfcl_solver.correct(*args)['samples']==result['samples'] and len(seen)==4 forged=copy.deepcopy(result['samples'][0]);forged['target']['content']='[]' with pytest.raises(ValueError):validate_sample(forged,{('bfcl',row['task_id']):row}) def test_multiturn_prefix_enters_student_first_solver_and_training_proofs(monkeypatch,tmp_path,config): from benchmarks.bfcl_multiturn_samples import compile_task from benchmarks.bfcl_solver import run row=multi_row();initial=state(tmp_path) monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) run_episode(row,initial,config,tmp_path/'failed') proposal={'category':row['category'],'end_turn':1};nonce=['fixture'];task=compile_task(row,proposal,nonce) write_json(tmp_path/'author.json',{'backend':'teacher','proposal':proposal,'response':{'content':json.dumps(proposal)}}) answer_from_trace(monkeypatch,task) result=run(initial,task,row,row['tools'],proposal,nonce,tmp_path/'failed/outcome.json',tmp_path/'author.json', config,tmp_path/'solver',tmp_path/'run',[],deadline=time.monotonic()+100) assert len(result['samples'])==2 and result['source_history']==['student_initial'] for s in result['samples']:assert validate_sample(s,{('bfcl',row['task_id']):row}) def test_harness_keeps_verified_multiturn_demonstrations(monkeypatch,tmp_path,config): from operators.harness_v2 import HarnessSessionStrategy row=multi_row();initial=state(tmp_path) monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) old=run_episode(row,initial,config,tmp_path/'failed');answer_from_trace(monkeypatch,row) good=run_episode(row,initial,config,tmp_path/'good') root=tmp_path/'nested';write_json(root/'baseline/0/stage-result.json',{'results':[old]}) strategy=object.__new__(HarnessSessionStrategy);strategy.config=config;strategy.run_root=tmp_path/'run' session=SimpleNamespace(panel=[row],journal=SimpleNamespace(root=root),root=root) strategy.retain_demonstrations(session,{'candidate':initial,'index':0},[good]) samples=read_jsonl(next((tmp_path/'run/harness-demonstrations').glob('*.jsonl'))) assert len(samples)==4 for s in samples:assert validate_sample(s,{('bfcl',row['task_id']):row}) def test_eight_category_weights_and_heldout_barriers(): from benchmarks.training import balanced_weights from benchmarks.robust_gate import assess,selection_settings cfg=read_json(ROOT/'plan/bfcl-only/full.json') rows=[static_row(i,c) if c not in mt.CATEGORIES else multi_row(c,index=i) for i,c in enumerate(CATEGORIES)] samples=[s for r in rows for s in replay_samples(r)] weights=balanced_weights(samples,cfg['training']['source_weights']) for c in CATEGORIES:assert sum(w for s,w in zip(samples,weights) if s['category']==c)==pytest.approx(.125) verdict=assess(rows,[0,0,0,0,1,0,0,0],selection_settings(cfg),cfg['seed'],{'bfcl/'+c:1 for c in CATEGORIES}) assert verdict['accepted'] and verdict['delta']==pytest.approx(.125) for role in ('selection','audit','meta_eval','sealed_test'): with pytest.raises(ValueError):replay_samples(multi_row(role=role)) from harness.interfaces import merge_contexts from benchmarks.runtime import STATIC_POLICY merged=merge_contexts([('bfcl','static',{'policy':STATIC_POLICY,'tools':{}}),('bfcl','multi',{'policy':mt.POLICY,'tools':{}})]) assert set(merged['bfcl']['case_policies'])=={'static','multi'} def test_multiturn_targets_run_actual_cpu_optimizer(tmp_path): torch=pytest.importorskip('torch') from modeling.train import PoolDataset,collate,optimize,validate_training_admission from benchmarks.training import balanced_weights samples=[s for c in CATEGORIES for s in replay_samples(multi_row(c) if c in mt.CATEGORIES else static_row(category=c))] validate_training_admission(samples) class Tokenizer: def apply_chat_template(self,messages,add_generation_prompt=False,**kw): return ''.join('<'+m['role']+'>'+m.get('content','')+'' for m in messages)+('' if add_generation_prompt else '') def __call__(self,text,**kw):return {'input_ids':[1+ord(c)%63 for c in text]} ds=PoolDataset(samples,Tokenizer(),65536) assert len(ds)==len(samples) and ds.dropped==0 class Tiny(torch.nn.Module): def __init__(self): super().__init__();self.embedding=torch.nn.Embedding(80,8);self.head=torch.nn.Linear(8,80) def forward(self,input_ids,attention_mask=None,logits_to_keep=None): hidden=self.embedding(input_ids) if logits_to_keep is not None:hidden=hidden[:,logits_to_keep] return SimpleNamespace(logits=self.head(hidden)) torch.manual_seed(31);model=Tiny();before=model.head.weight.detach().clone() loader=torch.utils.data.DataLoader(ds,batch_size=1,shuffle=False,collate_fn=lambda b:collate(b,0)) args=SimpleNamespace(batch=2,max_steps=0,epochs=1,warmup_ratio=.05,lr=.01) receipt,*_=optimize(model,ds,loader,loader,args,tmp_path,{},'cpu') assert receipt['optimizer_steps']==(len(samples)+1)//2 and not torch.equal(before,model.head.weight) assert set(receipt['consumed'])=={s['sample_id'] for s in samples} assert (tmp_path/'training-observations.json').is_file() def test_complete_eight_category_data_preparation(monkeypatch,tmp_path): from common import is_validation_group from benchmarks.bfcl_data import run from benchmarks.scoring import canonical_answer from benchmarks.runtime import prompt from controller import decision_router from learning.trajectories import validate_pool cfg=read_json(ROOT/'plan/bfcl-only/full.json') cfg['qualification'].update(min_samples=16,min_tasks=8,min_domains=1,min_failure_types=1) cfg['data_training'].update(max_seconds=120,min_new_verified_train_samples=1) gen=cfg['bfcl_only']['generation'];gen.update(minimum_samples_per_category=2,minimum_groups_per_category=2, minimum_correction_samples=1,minimum_correction_groups=1) gen['correction']['max_seconds']=120 rows=[];failed_rows=[] for c in CATEGORIES: candidates=[multi_row(c,index=i) if c in mt.CATEGORIES else static_row(i,c) for i in range(70)] pair=[next(r for r in candidates if is_validation_group(r['split_group'])==side) for side in (False,True)] rows.extend(pair);failed_rows.append(pair[0]) initial=state(tmp_path);evidence=[] monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) for r in failed_rows: directory=tmp_path/'failed'/r['task_id'];result=run_episode(r,initial,cfg,directory) assert not result['resolved'] evidence.append({'domain':'bfcl','task_id':r['task_id'],'episode_dir':str(directory),'failure_labels':result['failure_labels'],'benchmark_resolved':False}) lookup={} for r in rows: if mt.is_multiturn(r): for step in mt.replay_trace(r)['steps']:lookup[digest(step['messages'])]=step['response'] else:lookup[digest(prompt(r))]=mt.canonical_message(canonical_answer(r)) def answer(messages,**kw):return copy.deepcopy(lookup[digest(messages)]),{'completion_tokens':2},digest(messages) monkeypatch.setattr(client,'chat',answer) def no_teacher(*a,**k):raise AssertionError('Corrections and replay already cover this fixture') monkeypatch.setattr(client,'data_chat',no_teacher) choose=decision_router.choose def student(*a,**kw):kw['legal_actions']=['student_retry'];return choose(*a,**kw) monkeypatch.setattr(decision_router,'choose',student) candidate,summary=run(initial,{'evidence':evidence},rows,None,cfg,tmp_path/'preparation',run_root=tmp_path/'run') assert summary['readiness']['ready'] samples=read_jsonl(candidate['data_path']);validate_pool(samples,rows) assert {s['category'] for s in samples}==set(CATEGORIES) provenance=read_json(summary['source_provenance']['path']) assert provenance['actions'] and not provenance['unmatched_source_claims'] def test_native_tool_call_feedback_is_valid_chat_history(monkeypatch,tmp_path,config): row=multi_row();initial=state(tmp_path) steps=mt.replay_trace(row)['steps'];queue=[] for i,s in enumerate(steps): if s['calls']: c=s['calls'][0];queue.append({'role':'assistant','content':None,'tool_calls':[{'id':'tool-'+str(i),'type':'function', 'function':{'name':c['name'],'arguments':json.dumps(c['arguments'])}}]}) else:queue.append(s['response']) it=iter(queue);seen=[] def answer(messages,**kw):seen.append(copy.deepcopy(messages));return next(it),{'completion_tokens':1},'native' monkeypatch.setattr(client,'chat',answer) assert run_episode(row,initial,config,tmp_path/'native')['resolved'] assert any(m.get('role')=='tool' for history in seen for m in history) assert mt.verify_trace(row,read_json(tmp_path/'native/multi-turn-generation.json'))['valid'] def test_eight_category_batch_evaluation_and_cached_resume(monkeypatch,tmp_path): from evaluation import runner from benchmarks.runtime import prompt from benchmarks.scoring import canonical_answer cfg=read_json(ROOT/'plan/bfcl-only/full.json');cfg['parallel_evaluation']['enabled']=False rows=[multi_row(c,role='selection',index=i) if c in mt.CATEGORIES else static_row(i,c,role='selection') for i,c in enumerate(CATEGORIES)] answers={} for row in rows: if mt.is_multiturn(row): for step in mt.replay_trace(row)['steps']:answers[digest(step['messages'])]=step['response'] else:answers[digest(prompt(row))]=mt.canonical_message(canonical_answer(row)) calls=[] def chat(messages,**kw):calls.append(digest(messages));return copy.deepcopy(answers[digest(messages)]),{'completion_tokens':2},digest(messages) monkeypatch.setattr(client,'chat',chat) monkeypatch.setattr(client,'ensure_served',lambda *a:None) monkeypatch.setattr(client,'verify_serving_profile',lambda *a:{'fixture':True}) initial=state(tmp_path);out=tmp_path/'evaluation' result=runner.evaluate(rows,initial,cfg,out) assert result['complete'] and result['resolved']==8 and result['score']==1. n=len(calls);assert n>8 and result['output_tokens']==2*n assert runner.evaluate(rows,initial,cfg,out)==result and len(calls)==n assert all('bfcl_diagnosis' not in r['detail'] for r in result['results'])