#!/usr/bin/env python3 """Exact-set completion + provenance check for one eval output dir (used by the worker, 05's requeue and 06_grade.py). Expected row ids per file come from D/SUBSAMPLE_MANIFEST.json (kept_rids_per_behaviour, cross-checked against D/*.jsonl) or, for a plain source dir (selection set), from D/*.jsonl directly. A row counts only if its model_tag ends with "__" of the adapter that was supposed to be evaluated. Prints one line per file (ok/missing/extra/badsha/dup) and a summary; exit 0 iff every file has exactly the expected id set with the right sha, no extra ids and no id twice (a duplicate id = two writers appended = incomplete, AUDIT_live_B 2). Usage: check_rows.py [--json] [--quiet]""" import json, os, sys, glob D, OUT, SHA = sys.argv[1:4]; JS = "--json" in sys.argv; Q = "--quiet" in sys.argv def rid(r): return str(r.get("row_uid", r.get("qid"))) def load(p): for l in open(p): if l.strip(): try: yield json.loads(l) except Exception: pass files = sorted(os.path.basename(f) for f in glob.glob(f"{D}/*.jsonl")) assert files, f"no *.jsonl under {D}" exp = {f: {rid(r) for r in load(f"{D}/{f}")} for f in files} mp = f"{D}/SUBSAMPLE_MANIFEST.json" if os.path.exists(mp): kept = json.load(open(mp))["kept_rids_per_behaviour"] for f in files: m = set(kept[f.split("__")[0]]); assert m == exp[f], f"subsample manifest disagrees with {f} ({len(m)} vs {len(exp[f])})" res = {}; ok = True for f in files: p = f"{OUT}/{f}"; got = set(); bad = 0; extra = set(); dup = 0 if os.path.exists(p): for r in load(p): i = rid(r) if not str(r.get("model_tag", "")).endswith("__" + SHA): bad += 1; continue if i in got or i in extra: dup += 1; continue (got if i in exp[f] else extra).add(i) miss = exp[f] - got; st = "ok" if not miss and not extra and not bad and not dup else "incomplete" ok &= st == "ok"; res[f] = {"expected": len(exp[f]), "have": len(got), "missing": len(miss), "extra": len(extra), "badsha": bad, "dup": dup, "status": st} if not Q: print(f"{f:50s} exp {len(exp[f]):5d} have {len(got):5d} missing {len(miss):5d} extra {len(extra):3d} badsha {bad:4d} dup {dup:3d} {st}") tot = sum(v["expected"] for v in res.values()); have = sum(v["have"] for v in res.values()) summary = {"complete": ok, "files": len(files), "expected_rows": tot, "rows_ok": have, "sha256": SHA, "out": OUT, "per_file": res} print(json.dumps(summary if JS else {k: v for k, v in summary.items() if k != "per_file"})) sys.exit(0 if ok else 1)