| """The design check that every experiment request has to pass. Run from the repo root: |
| |
| python -m unittest scripts/test_check_design.py |
| """ |
| import subprocess |
| import sys |
| import tempfile |
| import unittest |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parent)) |
|
|
| from check_design import check |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| SCRIPT = ROOT / "scripts" / "check_design.py" |
|
|
| FILLED = """# Does the LoRA cut divergence? |
| |
| ## Claim |
| |
| If we fine-tune Qwen3-4B on data/final_v4, its divergence rate on ConspiracyBench drops by at least 5 points |
| compared with the base model. |
| |
| ## Unit and metric |
| |
| One corpus item (315), k=5 samples each. Divergence rate = diverge verdicts / all verdicts, by pplx-decider. |
| |
| ## Controls |
| |
| The base model, and a LoRA trained on the same number of shuffled generic chat rows. Always-hold scores 0%. |
| |
| ## Data |
| |
| Corpus at commit 58de878c2. SFT set filtered (>=40-char replies, no slurs): 43.6k -> 24.4k rows. No SFT row |
| mentions a corpus claim; nothing from the corpus is trained on. |
| |
| ## Noise |
| |
| Three seeds, and a bootstrap over claim_id for a 95% interval. |
| |
| ## Readout |
| |
| A drop under 5 points, or an interval that covers 0, refutes the claim. |
| """ |
|
|
|
|
| class Check(unittest.TestCase): |
| def test_a_filled_design_passes(self): |
| self.assertEqual(check(FILLED), []) |
|
|
| def test_the_template_fails_every_stage(self): |
| errors = {f["stage"] for f in check((ROOT / "experiment_requests" / "TEMPLATE.md").read_text(encoding="utf-8")) |
| if f["level"] == "error"} |
| self.assertEqual(errors, {"claim", "metric", "controls", "data", "noise", "readout"}) |
|
|
| def test_controls_must_name_a_baseline(self): |
| text = FILLED.replace("The base model, and a LoRA trained on the same number of shuffled generic chat rows. " |
| "Always-hold scores 0%.", "We compare the two arms.") |
| self.assertIn(("error", "controls"), {(f["level"], f["stage"]) for f in check(text)}) |
|
|
| def test_exit_code(self): |
| with tempfile.TemporaryDirectory() as d: |
| good, bad = Path(d) / "good.md", Path(d) / "bad.md" |
| good.write_text(FILLED, encoding="utf-8") |
| bad.write_text(FILLED.replace("## Readout", "## Results"), encoding="utf-8") |
| run = lambda p: subprocess.run([sys.executable, "-I", str(SCRIPT), str(p)], capture_output=True, text=True) |
| self.assertEqual(run(good).returncode, 0) |
| out = run(bad) |
| self.assertEqual(out.returncode, 1) |
| self.assertIn("no `Readout` section", out.stdout) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
