File size: 5,273 Bytes
1831ecd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | """Run the actual RSI controller/paired gate/checkpoint/audit with offline evaluator doubles."""
import copy
from pathlib import Path
from common import digest,read_json,read_jsonl,write_json,paired_episode_seed
from helpers import state
from test_release_integration import command,ROOT
def test_controller_accept_reject_resume_and_audit_isolation(monkeypatch,tmp_path,config):
import rsi_loop
from harness import manager
from controller import seed_service
from contextlib import contextmanager
cfg=copy.deepcopy(config);cfg.update(iterations=3,operators=['harness'])
config_path=tmp_path/'input.json';write_json(config_path,cfg)
campaign=tmp_path/'campaign'
prepared=command(ROOT/'prepare.py','--config',config_path,'--output',campaign)
assert prepared.returncode==0,prepared.stdout+prepared.stderr
cfg=read_json(campaign/'config.json')
selection=read_jsonl(campaign/'partition/selection.jsonl')
by_category={c:[r for r in selection if r['category']==c] for c in cfg['bfcl_only']['categories']}
base_ids={r['task_id'] for rows in by_category.values() for r in rows[:3]}
gain0=(base_ids-{by_category['multiple'][0]['task_id']})|{
by_category['simple_python'][3]['task_id'],by_category['simple_python'][4]['task_id']}
gain2=gain0|{by_category['simple_python'][5]['task_id']}
paths=[]
for i in range(3):
artifact=manager.initial_artifact();artifact['instructions_extra']=f'Check declared argument structure; revision {i}.'
paths.append(str(manager.save_version(tmp_path/f'candidate-{i}',artifact)))
proposals=[];analyzed_roles=[];evaluation_roles=[]
def evaluate(rows,current,effective,directory):
roles={r['split_role'] for r in rows};assert len(roles)==1
role=next(iter(roles));evaluation_roles.append(role)
assert role in ('improve','selection','audit') # sealed/meta never opened by the loop.
if role=='selection':
ids=base_ids if current['harness'] not in paths else [gain0,set(),gain2][paths.index(current['harness'])]
else:ids=set() # Deliberately useless audit scores cannot veto acceptance.
results=[{'domain':'bfcl','task_id':r['task_id'],'resolved':r['task_id'] in ids,
'det_reward':float(r['task_id'] in ids),'failure_labels':[] if r['task_id'] in ids else ['bfcl_argument_value'],
'split_role':role,'seed':paired_episode_seed(r,effective),'output_tokens':0,
'episode_dir':str(Path(directory)/r['task_id'])} for r in rows]
score=sum(r['resolved'] for r in results)/len(results)
return {'complete':True,'results':results,'score':score,'output_tokens':0,
'by_domain':{'bfcl':{'score':score,'resolved':sum(r['resolved'] for r in results),'total':len(results)}}}
def analyze(results,output,**kwargs):
analyzed_roles.extend(r['split_role'] for r in results)
value={'counts':{'bfcl_argument_value':len(results)},'summary':'Observed improve argument errors.',
'evidence':[{'domain':'bfcl','task_id':r['task_id'],'split_role':'improve',
'failure_labels':['bfcl_argument_value'],'benchmark_resolved':False} for r in results[:12]]}
write_json(output,value);return value
def search(operator,current,diagnosis,rows,effective,directory,iteration,*args,**kwargs):
assert operator=='harness' and all(r['split_role']=='improve' for r in rows)
assert 'audit' not in diagnosis
proposals.append(copy.deepcopy(current))
return {**current,'harness':paths[iteration]},{'suboperator':'prompt_update','proposal':{},
'nested':{'status':'candidate_ready','trials':[],'cost':{'output_tokens':0}}}
@contextmanager
def replicas(*args,**kwargs):yield [],['Offline controller integration fixture']
monkeypatch.setattr(seed_service,'replicas',replicas)
monkeypatch.setattr(rsi_loop.runner,'evaluate',evaluate)
monkeypatch.setattr(rsi_loop.failure_analyzer,'analyze',analyze)
monkeypatch.setattr(rsi_loop.nested_rsi,'search',search)
monkeypatch.setattr(rsi_loop.model_client,'verify_serving_profile',lambda req:{'fixture':True})
monkeypatch.setattr('modeling.service_probe.check_generation',lambda *args:{'fixture':True})
output=campaign/'run'
rsi_loop._run(cfg,output)
history=read_json(output/'history.json')
assert [r['accepted'] for r in history]==[True,False,True]
assert all(r['selection_comparison_status']=='completed' for r in history)
assert proposals[1]['harness']==paths[0] and proposals[2]['harness']==paths[0]
assert history[0]['gate']['robust']['by_source']['bfcl/multiple']<0
assert history[0]['gate']['acceptance_rule']=='source_balanced_positive_mean'
assert read_json(output/'controller-state.json')['state']['harness']==paths[2]
assert read_json(output/'status.json')['status']=='pilot_iterations_complete'
curve=read_json(output/'audit-curve.json')
assert curve['used_for_any_decision'] is False and len(curve['iterations'])==3
assert set(analyzed_roles)=={'improve'} and set(evaluation_roles)=={'improve','selection','audit'}
count=len(proposals);rsi_loop._run(cfg,output)
assert len(proposals)==count and read_json(output/'history.json')==history
|