snf_experiments / method_v2 /evidence /code /suite /tests /test_method_v2.py
CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
3.2 kB
import json
import tempfile
import unittest
from pathlib import Path
from suite.create_method_tasks import create
from suite.design import job_id
from suite.method_v2 import plan_study
from suite.worker import audit_control, common_v2_job, v2_baseline_key
class MethodV2Tests(unittest.TestCase):
def test_predeclared_tasks_split_and_no_prompt_reuse(self):
tasks=create(); self.assertEqual(len(tasks),32)
self.assertEqual(len({t['id'] for t in tasks}),32)
self.assertEqual(len({t['prompt'] for t in tasks}),32)
self.assertTrue(all(t.get('confirmatory') == (t['split']=='test') for t in tasks))
def test_plan_is_immutable_and_has_same_budget_recipes(self):
with tempfile.TemporaryDirectory() as d:
tasks=Path(d)/'tasks.jsonl'
tasks.write_text(''.join(json.dumps(t)+'\n' for t in create()))
a=plan_study(tasks,Path(d)/'dev','dev')
b=plan_study(tasks,Path(d)/'dev','dev')
self.assertEqual(a,b);self.assertEqual(a['planned'],2772)
jobs=[json.loads(s) for s in (Path(d)/'dev/jobs.jsonl').read_text().splitlines()]
self.assertTrue(all(j['task']['split']=='dev' for j in jobs))
self.assertTrue(all(j['id']==job_id({k:v for k,v in j.items() if k!='id'}) for j in jobs))
self.assertTrue(all('steps' not in j for j in jobs))
with self.assertRaises(ValueError):plan_study(tasks,Path(d)/'dev','dev',alpha_points=17)
def test_baseline_identity_includes_new_sampler_controls(self):
base=dict(alpha=.5,arm='confidence',control_schedule='duration',reference_copy=True,
authority_weight=.5,task={'id':'x'},seed=3,method_id='m')
key=v2_baseline_key(base)
self.assertEqual(key,v2_baseline_key(dict(base,alpha=.9,requested_effect=.7,calibration_mode='mapped')))
for changed in [dict(control_schedule='amplitude'),dict(reference_copy=False),dict(authority_weight=1.)]:
self.assertNotEqual(key,v2_baseline_key(dict(base,**changed)))
def test_common_anchor_resets_all_method_controls(self):
j=common_v2_job(dict(alpha=.5,arm='control_aware',seed=3,control_schedule='amplitude',reference_copy=False,
context_fraction=.25,commitment_delay=.2,task={'id':'x'}))
self.assertEqual(j['alpha'],0);self.assertEqual(j['arm'],'confidence')
self.assertEqual(j['control_schedule'],'duration');self.assertTrue(j['reference_copy'])
self.assertEqual(j['context_fraction'],1.);self.assertEqual(j['commitment_delay'],0.)
def test_dose_audit_uses_target_and_step_denominators(self):
trace=[dict(target_count=4,target_direct_copy_count=1,target_temporary_count=3,
target_newly_count=2,progress=0.),
dict(target_count=4,target_direct_copy_count=1,target_temporary_count=1,
target_newly_count=2,progress=.5)]
out=audit_control(trace)
self.assertEqual(out['direct_copy_fraction'],.5)
self.assertEqual(out['exposure_per_target_step'],.5)
self.assertEqual(out['mean_target_commit_progress'],.25)
if __name__ == '__main__': unittest.main()