| import copy |
| import pytest |
| from common import read_json,write_json |
| from harness.interfaces import merge_contexts |
| from operators.harness_rsi import observed_domain_contexts,validate_proposal_shape,SUBOPERATORS |
| from helpers import row,state,episode |
|
|
|
|
| def test_context_union_keeps_second_case_tool(monkeypatch,tmp_path,config): |
| initial=state(tmp_path) |
| first=row();second=row(1) |
| second['tools'][0]['function']['name']='second_tool' |
| second['expected_calls'][0]['name']='second_tool' |
| second['bfcl_original_tools'][0]['name']='second_tool' |
| episode(monkeypatch,first,initial,config,tmp_path/'first',[]) |
| episode(monkeypatch,second,initial,config,tmp_path/'second',[]) |
| diagnosis={'evidence':[{'domain':'bfcl','task_id':r['task_id'],'episode_dir':str(tmp_path/d)} |
| for r,d in [(first,'first'),(second,'second')]]} |
| contexts=observed_domain_contexts(diagnosis) |
| assert set(contexts['bfcl']['tools'])=={'foo','second_tool'} |
| validate_proposal_shape({'suboperator':'tool_update','target_domains':['bfcl'], |
| 'change':{'second_tool':'Use arguments from the current specification.'}},contexts, |
| {'suboperators':SUBOPERATORS}) |
|
|
|
|
| def test_same_name_schema_conflict_is_preserved_and_rejected(): |
| a={'policy':'p','tools':{'f':'compute'},'tool_parameters':{'f':{'type':'integer'}}} |
| b=copy.deepcopy(a);b['tool_parameters']['f']={'type':'string'} |
| contexts=merge_contexts([('bfcl','a',a),('bfcl','b',b),('bfcl','again',a)]) |
| assert len(contexts['bfcl']['tool_variants']['f'])==2 |
| assert 'f' not in contexts['bfcl']['tool_parameters'] |
| with pytest.raises(ValueError,match='Ambiguous'): |
| validate_proposal_shape({'suboperator':'tool_update','target_domains':['bfcl'], |
| 'change':{'f':'always use integer'}},contexts,{'suboperators':SUBOPERATORS}) |
| assert a['tool_parameters']['f']=={'type':'integer'} |
|
|
|
|
| def test_policy_conflicts_and_held_out_evidence_fail_closed(): |
| with pytest.raises(ValueError,match='policies disagree'): |
| merge_contexts([('bfcl','a',{'policy':'p'}),('bfcl','b',{'policy':'q'})]) |
| with pytest.raises(ValueError,match='improve-only'): |
| observed_domain_contexts({'evidence':[{'domain':'bfcl','split_role':'audit'}]}) |
|
|
