DeciCo-RSI / tests /test_multiturn.py
juray0410's picture
Initial commit
1831ecd
Raw History Blame Contribute Delete
19.7 kB
"""Eight-category integration: real simulators/checkers, deterministic model fixtures.
No model service calls; the optimizer test uses actual CPU autograd. It does not
claim performance or GPU compatibility of the production Qwen weights.
"""
import copy
import json
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
import pytest
from common import digest, read_json, read_jsonl, write_json, file_sha256
from benchmarks import bfcl_multiturn as mt
from benchmarks.bfcl_partition import CATEGORIES, split
from benchmarks.formats import import_source
from benchmarks.runtime import run_episode
from benchmarks.bfcl_tasks import replay_samples, validate_sample
from modeling import client
from helpers import state, row as static_row
ROOT=Path(__file__).resolve().parents[1]
def multi_row(category='multi_turn_base', role='improve', index=0):
raw={'id':category+'_'+str(index),'question':[[{'role':'user','content':'Add 1 and 2.'}],
[{'role':'user','content':'Multiply the previous result by 2.'}]],'initial_config':{},'involved_classes':['MathAPI']}
truth=[['add(a=1,b=2)'],['multiply(a=3,b=2)']]
if category=='multi_turn_miss_param':
raw['question'][0][0]['content']='Add 1 to a value I will give you next.'
raw['question'][1][0]['content']='The value is 2.';truth=[[],['add(a=1,b=2)']]
if category=='multi_turn_miss_func':
raw['question'].insert(1,[]);raw['missed_function']={'1':['add']}
truth=[[],['add(a=1,b=2)'],['multiply(a=3,b=2)']]
raw['question'][0][0]['content']='Scenario '+category+' '+str(index)+'. '+raw['question'][0][0]['content']
r=mt.normalize(raw,category,truth)
r['tools']=[t for t in r['tools'] if t['function']['name'] in ('add','multiply')]
r.update(split_role=role,split_group=digest(['family',index]))
return r
def answer_from_trace(monkeypatch,row,extra='',backend='both'):
trace=mt.replay_trace(row,extra);answers=iter(trace['steps']);seen=[]
def answer(messages,**kwargs):
step=next(answers);assert messages==step['messages']
seen.append(copy.deepcopy(messages))
return copy.deepcopy(step['response']),{'completion_tokens':3},digest(messages)
if backend in ('both','student'):monkeypatch.setattr(client,'chat',answer)
if backend in ('both','teacher'):monkeypatch.setattr(client,'data_chat',answer)
return seen
@pytest.fixture(scope='module')
def imported():
spec=read_json(ROOT/'plan/bfcl-only/datasets.json')['sources'][0]
for e in spec['files']:
for k in ('path','answer_path'):e[k]=str(ROOT/e[k])
return import_source(spec)[0]
def test_all_799_multiturn_references_reexecute_and_validate(imported):
rows=[r for r in imported if mt.is_multiturn(r)]
assert len(rows)==799 and len(imported)==1784
for r in rows:
trace=mt.replay_trace(r)
assert mt.verify_trace(r,trace)['valid'],r['original_id']
from benchmarks.scoring import canonical_answer,matches
assert matches(r,canonical_answer(r)),r['original_id']
assert {r['category'] for r in imported}==set(CATEGORIES)
def test_quarantined_upstream_row_has_an_explicit_tool_schedule_conflict():
raw=[json.loads(l) for l in (ROOT/'data/source/bfcl-multi_turn_miss_func-questions.jsonl').read_text(encoding='utf-8').splitlines()]
raw=next(r for r in raw if r['id']=='multi_turn_miss_func_49')
truth=[json.loads(l) for l in (ROOT/'data/source/bfcl-multi_turn_miss_func-answers.jsonl').read_text(encoding='utf-8').splitlines()]
truth=next(r['ground_truth'] for r in truth if r['id']==raw['id'])
r=mt.normalize(raw,'multi_turn_miss_func',truth)
assert 'tail' not in {t['function']['name'] for t in mt.tools_at(r,1)}
assert any(c.startswith('tail(') for c in truth[1])
with pytest.raises(ValueError,match='Reference trajectory'):mt.replay_trace(r)
def test_split_keeps_all_scenario_variants_together(imported):
spec=read_json(ROOT/'plan/bfcl-only/datasets.json');cfg=read_json(ROOT/'plan/bfcl-only/full.json')
parts=split(imported,spec['role_minimum_groups_per_category'],cfg['seed'])
assert {k:len(v) for k,v in parts.items()}=={'improve':692,'selection':321,'meta_eval':129,'audit':321,'sealed_test':321}
families={}
for role,rows in parts.items():
assert {r['category'] for r in rows}==set(CATEGORIES)
for r in rows:
if mt.is_multiturn(r):families.setdefault(r['explicit_group'],set()).add((role,r['split_group']))
assert len(families)==200 and all(len(roles)==1 for roles in families.values())
assert parts==split(imported,spec['role_minimum_groups_per_category'],cfg['seed'])
@pytest.mark.parametrize('category',mt.CATEGORIES)
def test_runtime_is_stateful_turn_aligned_and_no_future_inputs(monkeypatch,tmp_path,config,category):
row=multi_row(category);initial=state(tmp_path);seen=answer_from_trace(monkeypatch,row)
result=run_episode(row,initial,config,tmp_path/'episode')
assert result['resolved'] and result['output_tokens']==len(seen)*3
trace=read_json(tmp_path/'episode/multi-turn-generation.json')
assert mt.verify_trace(row,trace)['valid']
assert row['turns'][-1] not in seen[0]
assert 'initial_config' not in json.dumps(seen) and 'expected_calls' not in json.dumps(seen)
if category=='multi_turn_miss_func':
assert '"name": "add"' not in seen[0][0]['content']
assert any('"name": "add"' in m[0]['content'] for m in seen[1:])
def test_simulators_are_isolated_and_cannot_execute_host_code(tmp_path):
row=multi_row();env=mt.Environment(row)
assert 'Error during execution' in env.execute([{'name':'__import__','arguments':{'name':'os'}}],set(env.methods))[0]
assert 'Error during execution' in env.execute([{'name':'_load_scenario','arguments':{}}],set(env.methods))[0]
for text in ['__import__("os").system("echo unsafe")','add(a=open("x"),b=2)','add(**{})']:
with pytest.raises((ValueError,SyntaxError)):mt.parse_reference(text)
def execute(_):
r=multi_row('multi_turn_long_context');return mt.verify_trace(r,mt.replay_trace(r))
with ThreadPoolExecutor(max_workers=4) as pool:assert all(r['valid'] for r in pool.map(execute,range(12)))
def test_parameter_mutation_does_not_modify_recorded_actions(imported):
r=next(r for r in imported if r['original_id']=='multi_turn_base_90')
trace=mt.replay_trace(r);before=copy.deepcopy(trace)
assert mt.verify_trace(r,trace)['valid'] and trace==before
def test_full_harness_schema_repair_is_replayed(monkeypatch,tmp_path,config):
from harness import manager
row=multi_row();initial=state(tmp_path);artifact=manager.initial_artifact()
artifact['verification'].update(schema_check=True,max_repairs=1)
initial['harness']=str(manager.save_version(tmp_path/'repair-harness',artifact))
trace=mt.replay_trace(row);messages=[{'role':'assistant','content':'[{"name":"add","arguments":{}}]'}]
messages += [s['response'] for s in trace['steps']]
queue=iter(messages)
monkeypatch.setattr(client,'chat',lambda *a,**k:(next(queue),{'completion_tokens':2},'repair'))
result=run_episode(row,initial,config,tmp_path/'episode')
assert result['resolved']
actual=read_json(tmp_path/'episode/multi-turn-generation.json')
assert len(actual['steps'][0]['repairs'])==1 and mt.verify_trace(row,actual)['valid']
@pytest.mark.parametrize('source',['student_retry','temporary_harness','teacher'])
def test_data_correction_uses_one_bayesian_action_for_many_targets(monkeypatch,tmp_path,config,source):
from controller import decision_router
from benchmarks import bfcl_solver
from harness import manager
row=multi_row();initial=state(tmp_path)
monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed'))
failed=run_episode(row,initial,config,tmp_path/'failed');assert not failed['resolved']
original=decision_router.choose
def choose(*a,**kw):kw['legal_actions']=[source];return original(*a,**kw)
monkeypatch.setattr(decision_router,'choose',choose)
extra=''
if source=='temporary_harness':
artifact=manager.initial_artifact();extra='Use actual tool returns in the next step.';artifact['instructions_extra']=extra
guided={**initial,'harness':str(manager.save_version(tmp_path/'guided',artifact))}
monkeypatch.setattr(bfcl_solver,'temporary_harness',lambda *a,**k:{'valid':True,'candidate':guided,'output_tokens':0})
seen=answer_from_trace(monkeypatch,row,extra)
args=(initial,row,tmp_path/'failed',failed,config,tmp_path/'correct',tmp_path/'run',[])
result=bfcl_solver.correct(*args)
assert result['complete'] and len(result['samples'])==4 and len(result['attempts'])==1
for s in result['samples']:assert validate_sample(s,{('bfcl',row['task_id']):row})
saved=read_json(tmp_path/'correct/attempt-1/result.json')
assert saved['measurement']['bernoulli_observations']==1
assert saved['measurement']['target_episode_count']==1
assert bfcl_solver.correct(*args)['samples']==result['samples'] and len(seen)==4
forged=copy.deepcopy(result['samples'][0]);forged['target']['content']='[]'
with pytest.raises(ValueError):validate_sample(forged,{('bfcl',row['task_id']):row})
def test_multiturn_prefix_enters_student_first_solver_and_training_proofs(monkeypatch,tmp_path,config):
from benchmarks.bfcl_multiturn_samples import compile_task
from benchmarks.bfcl_solver import run
row=multi_row();initial=state(tmp_path)
monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed'))
run_episode(row,initial,config,tmp_path/'failed')
proposal={'category':row['category'],'end_turn':1};nonce=['fixture'];task=compile_task(row,proposal,nonce)
write_json(tmp_path/'author.json',{'backend':'teacher','proposal':proposal,'response':{'content':json.dumps(proposal)}})
answer_from_trace(monkeypatch,task)
result=run(initial,task,row,row['tools'],proposal,nonce,tmp_path/'failed/outcome.json',tmp_path/'author.json',
config,tmp_path/'solver',tmp_path/'run',[],deadline=time.monotonic()+100)
assert len(result['samples'])==2 and result['source_history']==['student_initial']
for s in result['samples']:assert validate_sample(s,{('bfcl',row['task_id']):row})
def test_harness_keeps_verified_multiturn_demonstrations(monkeypatch,tmp_path,config):
from operators.harness_v2 import HarnessSessionStrategy
row=multi_row();initial=state(tmp_path)
monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed'))
old=run_episode(row,initial,config,tmp_path/'failed');answer_from_trace(monkeypatch,row)
good=run_episode(row,initial,config,tmp_path/'good')
root=tmp_path/'nested';write_json(root/'baseline/0/stage-result.json',{'results':[old]})
strategy=object.__new__(HarnessSessionStrategy);strategy.config=config;strategy.run_root=tmp_path/'run'
session=SimpleNamespace(panel=[row],journal=SimpleNamespace(root=root),root=root)
strategy.retain_demonstrations(session,{'candidate':initial,'index':0},[good])
samples=read_jsonl(next((tmp_path/'run/harness-demonstrations').glob('*.jsonl')))
assert len(samples)==4
for s in samples:assert validate_sample(s,{('bfcl',row['task_id']):row})
def test_eight_category_weights_and_heldout_barriers():
from benchmarks.training import balanced_weights
from benchmarks.robust_gate import assess,selection_settings
cfg=read_json(ROOT/'plan/bfcl-only/full.json')
rows=[static_row(i,c) if c not in mt.CATEGORIES else multi_row(c,index=i) for i,c in enumerate(CATEGORIES)]
samples=[s for r in rows for s in replay_samples(r)]
weights=balanced_weights(samples,cfg['training']['source_weights'])
for c in CATEGORIES:assert sum(w for s,w in zip(samples,weights) if s['category']==c)==pytest.approx(.125)
verdict=assess(rows,[0,0,0,0,1,0,0,0],selection_settings(cfg),cfg['seed'],{'bfcl/'+c:1 for c in CATEGORIES})
assert verdict['accepted'] and verdict['delta']==pytest.approx(.125)
for role in ('selection','audit','meta_eval','sealed_test'):
with pytest.raises(ValueError):replay_samples(multi_row(role=role))
from harness.interfaces import merge_contexts
from benchmarks.runtime import STATIC_POLICY
merged=merge_contexts([('bfcl','static',{'policy':STATIC_POLICY,'tools':{}}),('bfcl','multi',{'policy':mt.POLICY,'tools':{}})])
assert set(merged['bfcl']['case_policies'])=={'static','multi'}
def test_multiturn_targets_run_actual_cpu_optimizer(tmp_path):
torch=pytest.importorskip('torch')
from modeling.train import PoolDataset,collate,optimize,validate_training_admission
from benchmarks.training import balanced_weights
samples=[s for c in CATEGORIES for s in replay_samples(multi_row(c) if c in mt.CATEGORIES else static_row(category=c))]
validate_training_admission(samples)
class Tokenizer:
def apply_chat_template(self,messages,add_generation_prompt=False,**kw):
return ''.join('<'+m['role']+'>'+m.get('content','')+'</'+m['role']+'>' for m in messages)+('<assistant>' if add_generation_prompt else '')
def __call__(self,text,**kw):return {'input_ids':[1+ord(c)%63 for c in text]}
ds=PoolDataset(samples,Tokenizer(),65536)
assert len(ds)==len(samples) and ds.dropped==0
class Tiny(torch.nn.Module):
def __init__(self):
super().__init__();self.embedding=torch.nn.Embedding(80,8);self.head=torch.nn.Linear(8,80)
def forward(self,input_ids,attention_mask=None,logits_to_keep=None):
hidden=self.embedding(input_ids)
if logits_to_keep is not None:hidden=hidden[:,logits_to_keep]
return SimpleNamespace(logits=self.head(hidden))
torch.manual_seed(31);model=Tiny();before=model.head.weight.detach().clone()
loader=torch.utils.data.DataLoader(ds,batch_size=1,shuffle=False,collate_fn=lambda b:collate(b,0))
args=SimpleNamespace(batch=2,max_steps=0,epochs=1,warmup_ratio=.05,lr=.01)
receipt,*_=optimize(model,ds,loader,loader,args,tmp_path,{},'cpu')
assert receipt['optimizer_steps']==(len(samples)+1)//2 and not torch.equal(before,model.head.weight)
assert set(receipt['consumed'])=={s['sample_id'] for s in samples}
assert (tmp_path/'training-observations.json').is_file()
def test_complete_eight_category_data_preparation(monkeypatch,tmp_path):
from common import is_validation_group
from benchmarks.bfcl_data import run
from benchmarks.scoring import canonical_answer
from benchmarks.runtime import prompt
from controller import decision_router
from learning.trajectories import validate_pool
cfg=read_json(ROOT/'plan/bfcl-only/full.json')
cfg['qualification'].update(min_samples=16,min_tasks=8,min_domains=1,min_failure_types=1)
cfg['data_training'].update(max_seconds=120,min_new_verified_train_samples=1)
gen=cfg['bfcl_only']['generation'];gen.update(minimum_samples_per_category=2,minimum_groups_per_category=2,
minimum_correction_samples=1,minimum_correction_groups=1)
gen['correction']['max_seconds']=120
rows=[];failed_rows=[]
for c in CATEGORIES:
candidates=[multi_row(c,index=i) if c in mt.CATEGORIES else static_row(i,c) for i in range(70)]
pair=[next(r for r in candidates if is_validation_group(r['split_group'])==side) for side in (False,True)]
rows.extend(pair);failed_rows.append(pair[0])
initial=state(tmp_path);evidence=[]
monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed'))
for r in failed_rows:
directory=tmp_path/'failed'/r['task_id'];result=run_episode(r,initial,cfg,directory)
assert not result['resolved']
evidence.append({'domain':'bfcl','task_id':r['task_id'],'episode_dir':str(directory),'failure_labels':result['failure_labels'],'benchmark_resolved':False})
lookup={}
for r in rows:
if mt.is_multiturn(r):
for step in mt.replay_trace(r)['steps']:lookup[digest(step['messages'])]=step['response']
else:lookup[digest(prompt(r))]=mt.canonical_message(canonical_answer(r))
def answer(messages,**kw):return copy.deepcopy(lookup[digest(messages)]),{'completion_tokens':2},digest(messages)
monkeypatch.setattr(client,'chat',answer)
def no_teacher(*a,**k):raise AssertionError('Corrections and replay already cover this fixture')
monkeypatch.setattr(client,'data_chat',no_teacher)
choose=decision_router.choose
def student(*a,**kw):kw['legal_actions']=['student_retry'];return choose(*a,**kw)
monkeypatch.setattr(decision_router,'choose',student)
candidate,summary=run(initial,{'evidence':evidence},rows,None,cfg,tmp_path/'preparation',run_root=tmp_path/'run')
assert summary['readiness']['ready']
samples=read_jsonl(candidate['data_path']);validate_pool(samples,rows)
assert {s['category'] for s in samples}==set(CATEGORIES)
provenance=read_json(summary['source_provenance']['path'])
assert provenance['actions'] and not provenance['unmatched_source_claims']
def test_native_tool_call_feedback_is_valid_chat_history(monkeypatch,tmp_path,config):
row=multi_row();initial=state(tmp_path)
steps=mt.replay_trace(row)['steps'];queue=[]
for i,s in enumerate(steps):
if s['calls']:
c=s['calls'][0];queue.append({'role':'assistant','content':None,'tool_calls':[{'id':'tool-'+str(i),'type':'function',
'function':{'name':c['name'],'arguments':json.dumps(c['arguments'])}}]})
else:queue.append(s['response'])
it=iter(queue);seen=[]
def answer(messages,**kw):seen.append(copy.deepcopy(messages));return next(it),{'completion_tokens':1},'native'
monkeypatch.setattr(client,'chat',answer)
assert run_episode(row,initial,config,tmp_path/'native')['resolved']
assert any(m.get('role')=='tool' for history in seen for m in history)
assert mt.verify_trace(row,read_json(tmp_path/'native/multi-turn-generation.json'))['valid']
def test_eight_category_batch_evaluation_and_cached_resume(monkeypatch,tmp_path):
from evaluation import runner
from benchmarks.runtime import prompt
from benchmarks.scoring import canonical_answer
cfg=read_json(ROOT/'plan/bfcl-only/full.json');cfg['parallel_evaluation']['enabled']=False
rows=[multi_row(c,role='selection',index=i) if c in mt.CATEGORIES else static_row(i,c,role='selection') for i,c in enumerate(CATEGORIES)]
answers={}
for row in rows:
if mt.is_multiturn(row):
for step in mt.replay_trace(row)['steps']:answers[digest(step['messages'])]=step['response']
else:answers[digest(prompt(row))]=mt.canonical_message(canonical_answer(row))
calls=[]
def chat(messages,**kw):calls.append(digest(messages));return copy.deepcopy(answers[digest(messages)]),{'completion_tokens':2},digest(messages)
monkeypatch.setattr(client,'chat',chat)
monkeypatch.setattr(client,'ensure_served',lambda *a:None)
monkeypatch.setattr(client,'verify_serving_profile',lambda *a:{'fixture':True})
initial=state(tmp_path);out=tmp_path/'evaluation'
result=runner.evaluate(rows,initial,cfg,out)
assert result['complete'] and result['resolved']==8 and result['score']==1.
n=len(calls);assert n>8 and result['output_tokens']==2*n
assert runner.evaluate(rows,initial,cfg,out)==result and len(calls)==n
assert all('bfcl_diagnosis' not in r['detail'] for r in result['results'])