Buckets:
| #!/usr/bin/env python3 | |
| """check_selection.py — DERIVE whether a public-200 lineage run is representative, from the | |
| attempted problem ids that are already in the manifest. No agent-declared fields required. | |
| Principle (human-cmpatino 20260804-112238): "derived beats declared." The manifest already | |
| carries the exact attempted ids, so the tier x label composition of any run — and hence | |
| whether its /200 rate is comparable to a full run — is derivable. We do NOT trust a declared | |
| `selection` block; we compute the truth from the ids and (if a `selection` block is present | |
| for human readability) reject it only when it CONTRADICTS the ids. | |
| What this gives the gate: a representativeness verdict that rides with the certs, computed the | |
| same way coverage2 does on the dashboard, so a tier/prefix-skewed /200 can't be read as a lane | |
| score — and it cannot be defeated by a mis-declared field, because there is no field to trust. | |
| Usage: check_selection.py <manifest.json> [bench.json] | |
| (bench.json defaults to ../scripts/public200/bench.json relative to this repo) | |
| Exit: 0 = ok (derived verdict reported; any declared selection matches the ids) | |
| 1 = a declared `selection` field CONTRADICTS the attempted ids (a lie) | |
| 2 = usage / cannot resolve ids against the bench | |
| """ | |
| from __future__ import annotations | |
| import json, os, sys | |
| from collections import Counter | |
| DEFAULT_BENCH = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "public200", "bench.json") | |
| def _proportional(cells_att: dict, cells_avail: dict, n_att: int, n_avail: int, tol: float = 1.0) -> bool: | |
| """Every available cell's attempted count within `tol` of its proportional expectation.""" | |
| if n_att <= 0 or n_avail <= 0: | |
| return False | |
| for cell, avail in cells_avail.items(): | |
| expected = n_att * (avail / n_avail) | |
| if abs(cells_att.get(cell, 0) - expected) > tol: | |
| return False | |
| return True | |
| def check(manifest: dict, bench_rows: list) -> tuple[int, list[str]]: | |
| bench = {r["id"]: r for r in bench_rows} | |
| cells_available = Counter(f"{r['tier']}/{r['label']}" for r in bench_rows) | |
| n_available = len(bench_rows) | |
| # Representativeness is a property of what was ATTEMPTED (the denominator's composition), | |
| # NOT what was solved. And `problem_ids` is not uniformly the attempted set across | |
| # producers: a subset-extraction manifest (foothold) records problem_ids = the SOLVED | |
| # rows while it actually attempted all 200 (source.attempted: 200); a per-run manifest | |
| # (toehold) records source.attempted_ids explicitly. So resolve the attempted set only | |
| # from authoritative signals, and ABSTAIN rather than mis-derive from solved ids. | |
| src = manifest.get("source") or {} | |
| attempted_ids, src_name = None, None | |
| if isinstance(src.get("attempted_ids"), list) and src["attempted_ids"]: | |
| attempted_ids, src_name = src["attempted_ids"], "source.attempted_ids" | |
| elif isinstance(src.get("attempted"), int) and src["attempted"] == n_available \ | |
| and "public200" in str(src.get("problems_from", "")).replace("/", "").replace("-", ""): | |
| # a full attempt of the bench: composition == the full bench, representative by construction | |
| attempted_ids, src_name = [r["id"] for r in bench_rows], "source.attempted==n_available (full bench sweep)" | |
| if attempted_ids is None: | |
| # Cannot prove what was attempted. `problem_ids`/`results` may be solved-only, and | |
| # calling a full run non-representative off the solved set is the harmful error. | |
| return 0, ["[abstain] cannot determine the ATTEMPTED set from this manifest " | |
| "(no source.attempted_ids, and source.attempted != n_available) — representativeness " | |
| "is undecidable here; `problem_ids` may be solved-only. Emit source.attempted_ids to enable the check."] | |
| unknown = [i for i in attempted_ids if i not in bench] | |
| if unknown: | |
| return 2, [f"[usage] {len(unknown)} attempted id(s) not in the public-200 bench, e.g. {unknown[:3]} — wrong bench?"] | |
| # ---- DERIVE the composition and the representativeness verdict from the ids ---- | |
| cells_attempted = Counter(f"{bench[i]['tier']}/{bench[i]['label']}" for i in attempted_ids) | |
| n_att = len(attempted_ids) | |
| is_full = n_att == n_available | |
| representative = is_full or _proportional(cells_attempted, cells_available, n_att, n_available) | |
| verdict = ("representative (/200 rate is lane-comparable)" if representative | |
| else "NON-representative — its /200 rate is a DIAGNOSTIC, not a lane score") | |
| msgs = [f"[derived from {src_name}] n_attempted={n_att}/{n_available} -> {verdict}", | |
| f" cells_attempted={dict(cells_attempted)}"] | |
| # ---- if a `selection` block was declared for readability, it must match the ids ---- | |
| sel = manifest.get("selection") | |
| code = 0 | |
| if isinstance(sel, dict): | |
| if "representative" in sel and bool(sel["representative"]) != representative: | |
| msgs.append(f"[REJECT] declared selection.representative={sel['representative']} but the attempted " | |
| f"ids derive representative={representative} — declared field contradicts the data") | |
| code = 1 | |
| dca = sel.get("cells_attempted") | |
| if isinstance(dca, dict) and Counter(dca) != cells_attempted: | |
| msgs.append(f"[REJECT] declared selection.cells_attempted disagrees with the ids " | |
| f"(declared={dict(dca)} vs derived={dict(cells_attempted)})") | |
| code = 1 | |
| if code == 0: | |
| msgs.append("[ok] declared `selection` block matches the derived composition") | |
| return code, msgs | |
| def main(argv): | |
| if len(argv) not in (2, 3): | |
| print("usage: check_selection.py <manifest.json> [bench.json]", file=sys.stderr) | |
| return 2 | |
| manifest = json.load(open(argv[1])) | |
| bench_path = argv[2] if len(argv) == 3 else DEFAULT_BENCH | |
| bench = json.load(open(bench_path)) | |
| bench_rows = bench["problems"] if isinstance(bench, dict) else bench | |
| code, msgs = check(manifest, bench_rows) | |
| for m in msgs: | |
| print(m) | |
| return code | |
| if __name__ == "__main__": | |
| sys.exit(main(sys.argv)) | |
Xet Storage Details
- Size:
- 6.24 kB
- Xet hash:
- 16cd0b4506dce09c092009a9a20abf0719f423ce53aeb6d75119484c444b03df
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.