CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
1.21 kB
import json
import tempfile
import unittest
from pathlib import Path
from suite.design import ROOT,read_tasks,plan,job_id
class DesignTests(unittest.TestCase):
def test_smoke_is_reproducible(self):
tasks=read_tasks(ROOT/'suite/tasks.example.jsonl');jobs=plan(tasks,'smoke')
self.assertEqual(jobs,plan(tasks,'smoke'));self.assertEqual(len(jobs),42)
self.assertEqual(len({j['id'] for j in jobs}),len(jobs))
def test_no_duplicate_task_inflation(self):
with self.assertRaises(ValueError):plan(read_tasks(ROOT/'suite/tasks.example.jsonl'),'pilot')
def test_dev_cannot_fill_test(self):
tasks=read_tasks(ROOT/'suite/tasks.example.jsonl')
for t in tasks:t['split']='dev'
with self.assertRaises(ValueError):plan(tasks,'smoke')
def test_identity_is_order_independent(self):self.assertEqual(job_id(dict(a=1,b=2)),job_id(dict(b=2,a=1)))
def test_empty_region_rejected(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d)/'tasks';p.write_text(json.dumps(dict(id='x',prompt='a',reference_prompt='b',target_time=[.8,.8])))
with self.assertRaises(ValueError):read_tasks(p)
if __name__=='__main__':unittest.main()