Download tests/test_multiturn.py from RL-MIND/DeciCo-RSI: direct link, hf CLI and curl.
- Browser
- Download file 19.7 kB
-
https://huggingface.co/RL-MIND/DeciCo-RSI/resolve/main/tests/test_multiturn.py
- Command line
-
hf download hf://RL-MIND/DeciCo-RSI/tests/test_multiturn.py
-
curl -L -o test_multiturn.py https://huggingface.co/RL-MIND/DeciCo-RSI/resolve/main/tests/test_multiturn.py
19.7 kB
| """Eight-category integration: real simulators/checkers, deterministic model fixtures. | |
| No model service calls; the optimizer test uses actual CPU autograd. It does not | |
| claim performance or GPU compatibility of the production Qwen weights. | |
| """ | |
| import copy | |
| import json | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| import pytest | |
| from common import digest, read_json, read_jsonl, write_json, file_sha256 | |
| from benchmarks import bfcl_multiturn as mt | |
| from benchmarks.bfcl_partition import CATEGORIES, split | |
| from benchmarks.formats import import_source | |
| from benchmarks.runtime import run_episode | |
| from benchmarks.bfcl_tasks import replay_samples, validate_sample | |
| from modeling import client | |
| from helpers import state, row as static_row | |
| ROOT=Path(__file__).resolve().parents[1] | |
| def multi_row(category='multi_turn_base', role='improve', index=0): | |
| raw={'id':category+'_'+str(index),'question':[[{'role':'user','content':'Add 1 and 2.'}], | |
| [{'role':'user','content':'Multiply the previous result by 2.'}]],'initial_config':{},'involved_classes':['MathAPI']} | |
| truth=[['add(a=1,b=2)'],['multiply(a=3,b=2)']] | |
| if category=='multi_turn_miss_param': | |
| raw['question'][0][0]['content']='Add 1 to a value I will give you next.' | |
| raw['question'][1][0]['content']='The value is 2.';truth=[[],['add(a=1,b=2)']] | |
| if category=='multi_turn_miss_func': | |
| raw['question'].insert(1,[]);raw['missed_function']={'1':['add']} | |
| truth=[[],['add(a=1,b=2)'],['multiply(a=3,b=2)']] | |
| raw['question'][0][0]['content']='Scenario '+category+' '+str(index)+'. '+raw['question'][0][0]['content'] | |
| r=mt.normalize(raw,category,truth) | |
| r['tools']=[t for t in r['tools'] if t['function']['name'] in ('add','multiply')] | |
| r.update(split_role=role,split_group=digest(['family',index])) | |
| return r | |
| def answer_from_trace(monkeypatch,row,extra='',backend='both'): | |
| trace=mt.replay_trace(row,extra);answers=iter(trace['steps']);seen=[] | |
| def answer(messages,**kwargs): | |
| step=next(answers);assert messages==step['messages'] | |
| seen.append(copy.deepcopy(messages)) | |
| return copy.deepcopy(step['response']),{'completion_tokens':3},digest(messages) | |
| if backend in ('both','student'):monkeypatch.setattr(client,'chat',answer) | |
| if backend in ('both','teacher'):monkeypatch.setattr(client,'data_chat',answer) | |
| return seen | |
| def imported(): | |
| spec=read_json(ROOT/'plan/bfcl-only/datasets.json')['sources'][0] | |
| for e in spec['files']: | |
| for k in ('path','answer_path'):e[k]=str(ROOT/e[k]) | |
| return import_source(spec)[0] | |
| def test_all_799_multiturn_references_reexecute_and_validate(imported): | |
| rows=[r for r in imported if mt.is_multiturn(r)] | |
| assert len(rows)==799 and len(imported)==1784 | |
| for r in rows: | |
| trace=mt.replay_trace(r) | |
| assert mt.verify_trace(r,trace)['valid'],r['original_id'] | |
| from benchmarks.scoring import canonical_answer,matches | |
| assert matches(r,canonical_answer(r)),r['original_id'] | |
| assert {r['category'] for r in imported}==set(CATEGORIES) | |
| def test_quarantined_upstream_row_has_an_explicit_tool_schedule_conflict(): | |
| raw=[json.loads(l) for l in (ROOT/'data/source/bfcl-multi_turn_miss_func-questions.jsonl').read_text(encoding='utf-8').splitlines()] | |
| raw=next(r for r in raw if r['id']=='multi_turn_miss_func_49') | |
| truth=[json.loads(l) for l in (ROOT/'data/source/bfcl-multi_turn_miss_func-answers.jsonl').read_text(encoding='utf-8').splitlines()] | |
| truth=next(r['ground_truth'] for r in truth if r['id']==raw['id']) | |
| r=mt.normalize(raw,'multi_turn_miss_func',truth) | |
| assert 'tail' not in {t['function']['name'] for t in mt.tools_at(r,1)} | |
| assert any(c.startswith('tail(') for c in truth[1]) | |
| with pytest.raises(ValueError,match='Reference trajectory'):mt.replay_trace(r) | |
| def test_split_keeps_all_scenario_variants_together(imported): | |
| spec=read_json(ROOT/'plan/bfcl-only/datasets.json');cfg=read_json(ROOT/'plan/bfcl-only/full.json') | |
| parts=split(imported,spec['role_minimum_groups_per_category'],cfg['seed']) | |
| assert {k:len(v) for k,v in parts.items()}=={'improve':692,'selection':321,'meta_eval':129,'audit':321,'sealed_test':321} | |
| families={} | |
| for role,rows in parts.items(): | |
| assert {r['category'] for r in rows}==set(CATEGORIES) | |
| for r in rows: | |
| if mt.is_multiturn(r):families.setdefault(r['explicit_group'],set()).add((role,r['split_group'])) | |
| assert len(families)==200 and all(len(roles)==1 for roles in families.values()) | |
| assert parts==split(imported,spec['role_minimum_groups_per_category'],cfg['seed']) | |
| def test_runtime_is_stateful_turn_aligned_and_no_future_inputs(monkeypatch,tmp_path,config,category): | |
| row=multi_row(category);initial=state(tmp_path);seen=answer_from_trace(monkeypatch,row) | |
| result=run_episode(row,initial,config,tmp_path/'episode') | |
| assert result['resolved'] and result['output_tokens']==len(seen)*3 | |
| trace=read_json(tmp_path/'episode/multi-turn-generation.json') | |
| assert mt.verify_trace(row,trace)['valid'] | |
| assert row['turns'][-1] not in seen[0] | |
| assert 'initial_config' not in json.dumps(seen) and 'expected_calls' not in json.dumps(seen) | |
| if category=='multi_turn_miss_func': | |
| assert '"name": "add"' not in seen[0][0]['content'] | |
| assert any('"name": "add"' in m[0]['content'] for m in seen[1:]) | |
| def test_simulators_are_isolated_and_cannot_execute_host_code(tmp_path): | |
| row=multi_row();env=mt.Environment(row) | |
| assert 'Error during execution' in env.execute([{'name':'__import__','arguments':{'name':'os'}}],set(env.methods))[0] | |
| assert 'Error during execution' in env.execute([{'name':'_load_scenario','arguments':{}}],set(env.methods))[0] | |
| for text in ['__import__("os").system("echo unsafe")','add(a=open("x"),b=2)','add(**{})']: | |
| with pytest.raises((ValueError,SyntaxError)):mt.parse_reference(text) | |
| def execute(_): | |
| r=multi_row('multi_turn_long_context');return mt.verify_trace(r,mt.replay_trace(r)) | |
| with ThreadPoolExecutor(max_workers=4) as pool:assert all(r['valid'] for r in pool.map(execute,range(12))) | |
| def test_parameter_mutation_does_not_modify_recorded_actions(imported): | |
| r=next(r for r in imported if r['original_id']=='multi_turn_base_90') | |
| trace=mt.replay_trace(r);before=copy.deepcopy(trace) | |
| assert mt.verify_trace(r,trace)['valid'] and trace==before | |
| def test_full_harness_schema_repair_is_replayed(monkeypatch,tmp_path,config): | |
| from harness import manager | |
| row=multi_row();initial=state(tmp_path);artifact=manager.initial_artifact() | |
| artifact['verification'].update(schema_check=True,max_repairs=1) | |
| initial['harness']=str(manager.save_version(tmp_path/'repair-harness',artifact)) | |
| trace=mt.replay_trace(row);messages=[{'role':'assistant','content':'[{"name":"add","arguments":{}}]'}] | |
| messages += [s['response'] for s in trace['steps']] | |
| queue=iter(messages) | |
| monkeypatch.setattr(client,'chat',lambda *a,**k:(next(queue),{'completion_tokens':2},'repair')) | |
| result=run_episode(row,initial,config,tmp_path/'episode') | |
| assert result['resolved'] | |
| actual=read_json(tmp_path/'episode/multi-turn-generation.json') | |
| assert len(actual['steps'][0]['repairs'])==1 and mt.verify_trace(row,actual)['valid'] | |
| def test_data_correction_uses_one_bayesian_action_for_many_targets(monkeypatch,tmp_path,config,source): | |
| from controller import decision_router | |
| from benchmarks import bfcl_solver | |
| from harness import manager | |
| row=multi_row();initial=state(tmp_path) | |
| monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) | |
| failed=run_episode(row,initial,config,tmp_path/'failed');assert not failed['resolved'] | |
| original=decision_router.choose | |
| def choose(*a,**kw):kw['legal_actions']=[source];return original(*a,**kw) | |
| monkeypatch.setattr(decision_router,'choose',choose) | |
| extra='' | |
| if source=='temporary_harness': | |
| artifact=manager.initial_artifact();extra='Use actual tool returns in the next step.';artifact['instructions_extra']=extra | |
| guided={**initial,'harness':str(manager.save_version(tmp_path/'guided',artifact))} | |
| monkeypatch.setattr(bfcl_solver,'temporary_harness',lambda *a,**k:{'valid':True,'candidate':guided,'output_tokens':0}) | |
| seen=answer_from_trace(monkeypatch,row,extra) | |
| args=(initial,row,tmp_path/'failed',failed,config,tmp_path/'correct',tmp_path/'run',[]) | |
| result=bfcl_solver.correct(*args) | |
| assert result['complete'] and len(result['samples'])==4 and len(result['attempts'])==1 | |
| for s in result['samples']:assert validate_sample(s,{('bfcl',row['task_id']):row}) | |
| saved=read_json(tmp_path/'correct/attempt-1/result.json') | |
| assert saved['measurement']['bernoulli_observations']==1 | |
| assert saved['measurement']['target_episode_count']==1 | |
| assert bfcl_solver.correct(*args)['samples']==result['samples'] and len(seen)==4 | |
| forged=copy.deepcopy(result['samples'][0]);forged['target']['content']='[]' | |
| with pytest.raises(ValueError):validate_sample(forged,{('bfcl',row['task_id']):row}) | |
| def test_multiturn_prefix_enters_student_first_solver_and_training_proofs(monkeypatch,tmp_path,config): | |
| from benchmarks.bfcl_multiturn_samples import compile_task | |
| from benchmarks.bfcl_solver import run | |
| row=multi_row();initial=state(tmp_path) | |
| monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) | |
| run_episode(row,initial,config,tmp_path/'failed') | |
| proposal={'category':row['category'],'end_turn':1};nonce=['fixture'];task=compile_task(row,proposal,nonce) | |
| write_json(tmp_path/'author.json',{'backend':'teacher','proposal':proposal,'response':{'content':json.dumps(proposal)}}) | |
| answer_from_trace(monkeypatch,task) | |
| result=run(initial,task,row,row['tools'],proposal,nonce,tmp_path/'failed/outcome.json',tmp_path/'author.json', | |
| config,tmp_path/'solver',tmp_path/'run',[],deadline=time.monotonic()+100) | |
| assert len(result['samples'])==2 and result['source_history']==['student_initial'] | |
| for s in result['samples']:assert validate_sample(s,{('bfcl',row['task_id']):row}) | |
| def test_harness_keeps_verified_multiturn_demonstrations(monkeypatch,tmp_path,config): | |
| from operators.harness_v2 import HarnessSessionStrategy | |
| row=multi_row();initial=state(tmp_path) | |
| monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) | |
| old=run_episode(row,initial,config,tmp_path/'failed');answer_from_trace(monkeypatch,row) | |
| good=run_episode(row,initial,config,tmp_path/'good') | |
| root=tmp_path/'nested';write_json(root/'baseline/0/stage-result.json',{'results':[old]}) | |
| strategy=object.__new__(HarnessSessionStrategy);strategy.config=config;strategy.run_root=tmp_path/'run' | |
| session=SimpleNamespace(panel=[row],journal=SimpleNamespace(root=root),root=root) | |
| strategy.retain_demonstrations(session,{'candidate':initial,'index':0},[good]) | |
| samples=read_jsonl(next((tmp_path/'run/harness-demonstrations').glob('*.jsonl'))) | |
| assert len(samples)==4 | |
| for s in samples:assert validate_sample(s,{('bfcl',row['task_id']):row}) | |
| def test_eight_category_weights_and_heldout_barriers(): | |
| from benchmarks.training import balanced_weights | |
| from benchmarks.robust_gate import assess,selection_settings | |
| cfg=read_json(ROOT/'plan/bfcl-only/full.json') | |
| rows=[static_row(i,c) if c not in mt.CATEGORIES else multi_row(c,index=i) for i,c in enumerate(CATEGORIES)] | |
| samples=[s for r in rows for s in replay_samples(r)] | |
| weights=balanced_weights(samples,cfg['training']['source_weights']) | |
| for c in CATEGORIES:assert sum(w for s,w in zip(samples,weights) if s['category']==c)==pytest.approx(.125) | |
| verdict=assess(rows,[0,0,0,0,1,0,0,0],selection_settings(cfg),cfg['seed'],{'bfcl/'+c:1 for c in CATEGORIES}) | |
| assert verdict['accepted'] and verdict['delta']==pytest.approx(.125) | |
| for role in ('selection','audit','meta_eval','sealed_test'): | |
| with pytest.raises(ValueError):replay_samples(multi_row(role=role)) | |
| from harness.interfaces import merge_contexts | |
| from benchmarks.runtime import STATIC_POLICY | |
| merged=merge_contexts([('bfcl','static',{'policy':STATIC_POLICY,'tools':{}}),('bfcl','multi',{'policy':mt.POLICY,'tools':{}})]) | |
| assert set(merged['bfcl']['case_policies'])=={'static','multi'} | |
| def test_multiturn_targets_run_actual_cpu_optimizer(tmp_path): | |
| torch=pytest.importorskip('torch') | |
| from modeling.train import PoolDataset,collate,optimize,validate_training_admission | |
| from benchmarks.training import balanced_weights | |
| samples=[s for c in CATEGORIES for s in replay_samples(multi_row(c) if c in mt.CATEGORIES else static_row(category=c))] | |
| validate_training_admission(samples) | |
| class Tokenizer: | |
| def apply_chat_template(self,messages,add_generation_prompt=False,**kw): | |
| return ''.join('<'+m['role']+'>'+m.get('content','')+'</'+m['role']+'>' for m in messages)+('<assistant>' if add_generation_prompt else '') | |
| def __call__(self,text,**kw):return {'input_ids':[1+ord(c)%63 for c in text]} | |
| ds=PoolDataset(samples,Tokenizer(),65536) | |
| assert len(ds)==len(samples) and ds.dropped==0 | |
| class Tiny(torch.nn.Module): | |
| def __init__(self): | |
| super().__init__();self.embedding=torch.nn.Embedding(80,8);self.head=torch.nn.Linear(8,80) | |
| def forward(self,input_ids,attention_mask=None,logits_to_keep=None): | |
| hidden=self.embedding(input_ids) | |
| if logits_to_keep is not None:hidden=hidden[:,logits_to_keep] | |
| return SimpleNamespace(logits=self.head(hidden)) | |
| torch.manual_seed(31);model=Tiny();before=model.head.weight.detach().clone() | |
| loader=torch.utils.data.DataLoader(ds,batch_size=1,shuffle=False,collate_fn=lambda b:collate(b,0)) | |
| args=SimpleNamespace(batch=2,max_steps=0,epochs=1,warmup_ratio=.05,lr=.01) | |
| receipt,*_=optimize(model,ds,loader,loader,args,tmp_path,{},'cpu') | |
| assert receipt['optimizer_steps']==(len(samples)+1)//2 and not torch.equal(before,model.head.weight) | |
| assert set(receipt['consumed'])=={s['sample_id'] for s in samples} | |
| assert (tmp_path/'training-observations.json').is_file() | |
| def test_complete_eight_category_data_preparation(monkeypatch,tmp_path): | |
| from common import is_validation_group | |
| from benchmarks.bfcl_data import run | |
| from benchmarks.scoring import canonical_answer | |
| from benchmarks.runtime import prompt | |
| from controller import decision_router | |
| from learning.trajectories import validate_pool | |
| cfg=read_json(ROOT/'plan/bfcl-only/full.json') | |
| cfg['qualification'].update(min_samples=16,min_tasks=8,min_domains=1,min_failure_types=1) | |
| cfg['data_training'].update(max_seconds=120,min_new_verified_train_samples=1) | |
| gen=cfg['bfcl_only']['generation'];gen.update(minimum_samples_per_category=2,minimum_groups_per_category=2, | |
| minimum_correction_samples=1,minimum_correction_groups=1) | |
| gen['correction']['max_seconds']=120 | |
| rows=[];failed_rows=[] | |
| for c in CATEGORIES: | |
| candidates=[multi_row(c,index=i) if c in mt.CATEGORIES else static_row(i,c) for i in range(70)] | |
| pair=[next(r for r in candidates if is_validation_group(r['split_group'])==side) for side in (False,True)] | |
| rows.extend(pair);failed_rows.append(pair[0]) | |
| initial=state(tmp_path);evidence=[] | |
| monkeypatch.setattr(client,'chat',lambda *a,**k:({'role':'assistant','content':'[]'},{'completion_tokens':1},'failed')) | |
| for r in failed_rows: | |
| directory=tmp_path/'failed'/r['task_id'];result=run_episode(r,initial,cfg,directory) | |
| assert not result['resolved'] | |
| evidence.append({'domain':'bfcl','task_id':r['task_id'],'episode_dir':str(directory),'failure_labels':result['failure_labels'],'benchmark_resolved':False}) | |
| lookup={} | |
| for r in rows: | |
| if mt.is_multiturn(r): | |
| for step in mt.replay_trace(r)['steps']:lookup[digest(step['messages'])]=step['response'] | |
| else:lookup[digest(prompt(r))]=mt.canonical_message(canonical_answer(r)) | |
| def answer(messages,**kw):return copy.deepcopy(lookup[digest(messages)]),{'completion_tokens':2},digest(messages) | |
| monkeypatch.setattr(client,'chat',answer) | |
| def no_teacher(*a,**k):raise AssertionError('Corrections and replay already cover this fixture') | |
| monkeypatch.setattr(client,'data_chat',no_teacher) | |
| choose=decision_router.choose | |
| def student(*a,**kw):kw['legal_actions']=['student_retry'];return choose(*a,**kw) | |
| monkeypatch.setattr(decision_router,'choose',student) | |
| candidate,summary=run(initial,{'evidence':evidence},rows,None,cfg,tmp_path/'preparation',run_root=tmp_path/'run') | |
| assert summary['readiness']['ready'] | |
| samples=read_jsonl(candidate['data_path']);validate_pool(samples,rows) | |
| assert {s['category'] for s in samples}==set(CATEGORIES) | |
| provenance=read_json(summary['source_provenance']['path']) | |
| assert provenance['actions'] and not provenance['unmatched_source_claims'] | |
| def test_native_tool_call_feedback_is_valid_chat_history(monkeypatch,tmp_path,config): | |
| row=multi_row();initial=state(tmp_path) | |
| steps=mt.replay_trace(row)['steps'];queue=[] | |
| for i,s in enumerate(steps): | |
| if s['calls']: | |
| c=s['calls'][0];queue.append({'role':'assistant','content':None,'tool_calls':[{'id':'tool-'+str(i),'type':'function', | |
| 'function':{'name':c['name'],'arguments':json.dumps(c['arguments'])}}]}) | |
| else:queue.append(s['response']) | |
| it=iter(queue);seen=[] | |
| def answer(messages,**kw):seen.append(copy.deepcopy(messages));return next(it),{'completion_tokens':1},'native' | |
| monkeypatch.setattr(client,'chat',answer) | |
| assert run_episode(row,initial,config,tmp_path/'native')['resolved'] | |
| assert any(m.get('role')=='tool' for history in seen for m in history) | |
| assert mt.verify_trace(row,read_json(tmp_path/'native/multi-turn-generation.json'))['valid'] | |
| def test_eight_category_batch_evaluation_and_cached_resume(monkeypatch,tmp_path): | |
| from evaluation import runner | |
| from benchmarks.runtime import prompt | |
| from benchmarks.scoring import canonical_answer | |
| cfg=read_json(ROOT/'plan/bfcl-only/full.json');cfg['parallel_evaluation']['enabled']=False | |
| rows=[multi_row(c,role='selection',index=i) if c in mt.CATEGORIES else static_row(i,c,role='selection') for i,c in enumerate(CATEGORIES)] | |
| answers={} | |
| for row in rows: | |
| if mt.is_multiturn(row): | |
| for step in mt.replay_trace(row)['steps']:answers[digest(step['messages'])]=step['response'] | |
| else:answers[digest(prompt(row))]=mt.canonical_message(canonical_answer(row)) | |
| calls=[] | |
| def chat(messages,**kw):calls.append(digest(messages));return copy.deepcopy(answers[digest(messages)]),{'completion_tokens':2},digest(messages) | |
| monkeypatch.setattr(client,'chat',chat) | |
| monkeypatch.setattr(client,'ensure_served',lambda *a:None) | |
| monkeypatch.setattr(client,'verify_serving_profile',lambda *a:{'fixture':True}) | |
| initial=state(tmp_path);out=tmp_path/'evaluation' | |
| result=runner.evaluate(rows,initial,cfg,out) | |
| assert result['complete'] and result['resolved']==8 and result['score']==1. | |
| n=len(calls);assert n>8 and result['output_tokens']==2*n | |
| assert runner.evaluate(rows,initial,cfg,out)==result and len(calls)==n | |
| assert all('bfcl_diagnosis' not in r['detail'] for r in result['results']) | |