| { |
| "columns": [ |
| "Eval set / metric", |
| "What it measures", |
| "Split-integrity rule", |
| "Statistics", |
| "Promotion threshold", |
| "Source", |
| "Confidence", |
| "Status" |
| ], |
| "widths": [ |
| 36, |
| 40, |
| 48, |
| 40, |
| 36, |
| 36, |
| 10, |
| 14 |
| ], |
| "rows": [ |
| [ |
| "miniF2F-valid (226, FILTERED: 13 unsupported, 4 elab failures, 1 overlap)", |
| "Contrast decisions K1-K4, averaging, stop rule", |
| "Driver refuses test-split scoring (test_guard); every selection logged", |
| "pass@8 dev; promotions confirmed with fresh-seed n=32", |
| "Heuristic gates only (+3 is not significant)", |
| "1c483cc; scripts/p2/expert_iteration.py", |
| "High", |
| "Done (built)" |
| ], |
| [ |
| "ProofNet-dev (to build)", |
| "Keeps selection from being miniF2F-only", |
| "Pinned, decontaminated", |
| "pass@8", |
| "-", |
| "docs/SOTA_PLAN.md Phase 0", |
| "Med", |
| "Planned" |
| ], |
| [ |
| "PRIMARY: miniF2F-test class A", |
| "Final candidate vs anchor 202/244", |
| "Touched once (--allow-test-selection --rounds 1)", |
| "n=k=32 solved/244; exact two-sided McNemar; wins/losses; paired CI; mean pass@1", |
| "Significant needs ~+9 net (losses 4 -> 13 wins) ~ 211/244; 207 is descriptive only", |
| "docs/SOTA_PLAN.md section 1", |
| "High", |
| "Planned" |
| ], |
| [ |
| "SECONDARY: class B miniF2F-test, class A ProofNet-test", |
| "Correction mode; transfer", |
| "Same harness both arms; trajectory ledger", |
| "Holm-corrected McNemar", |
| "-", |
| "docs/SOTA_PLAN.md section 1", |
| "High", |
| "Planned" |
| ], |
| [ |
| "EXPLORATORY: class C, PutnamBench class A pass@8", |
| "Agentic; hard transfer", |
| "Protocol tuple; compared only to agentic systems / own anchor", |
| "descriptive", |
| "-", |
| "docs/SOTA_PLAN.md section 1", |
| "Med", |
| "Planned" |
| ], |
| [ |
| "Verifier budget", |
| "Proof check wall-clock", |
| "300 s both arms; 1,200 s sensitivity; timeouts undecided", |
| "timeout counts per arm", |
| "-", |
| "1cc55e4", |
| "High", |
| "Done (protocol)" |
| ], |
| [ |
| "Rewritten-statement count", |
| "problem_check substitutes the reference statement", |
| "Reported per arm", |
| "count of accepted samples whose displayed statement differs", |
| "must not differ materially between arms", |
| "docs/P2.md 8c item 18", |
| "High", |
| "Planned" |
| ], |
| [ |
| "Verifier false-accept suite", |
| "Reward hacking resistance", |
| "Adversarial proofs never in training", |
| "0 accepts required", |
| "0/61 (P1); red-team 0 confirmed", |
| "654c20c", |
| "High", |
| "Running" |
| ] |
| ] |
| } |
