260926_matched_eval / check_rows.py
jprivera's picture
upload batch 1/7 from pod 38.80.152.76
43eb135 verified
Raw History Blame Contribute Delete
2.61 kB
#!/usr/bin/env python3
"""Exact-set completion + provenance check for one eval output dir (used by the worker, 05's requeue and 06_grade.py).
Expected row ids per file come from D/SUBSAMPLE_MANIFEST.json (kept_rids_per_behaviour, cross-checked against D/*.jsonl) or, for a
plain source dir (selection set), from D/*.jsonl directly. A row counts only if its model_tag ends with "__<sha256>" of the adapter
that was supposed to be evaluated. Prints one line per file (ok/missing/extra/badsha/dup) and a summary; exit 0 iff every file has exactly
the expected id set with the right sha, no extra ids and no id twice (a duplicate id = two writers appended = incomplete, AUDIT_live_B 2).
Usage: check_rows.py <D> <OUT_DIR> <sha256> [--json] [--quiet]"""
import json, os, sys, glob
D, OUT, SHA = sys.argv[1:4]; JS = "--json" in sys.argv; Q = "--quiet" in sys.argv
def rid(r): return str(r.get("row_uid", r.get("qid")))
def load(p):
for l in open(p):
if l.strip():
try: yield json.loads(l)
except Exception: pass
files = sorted(os.path.basename(f) for f in glob.glob(f"{D}/*.jsonl"))
assert files, f"no *.jsonl under {D}"
exp = {f: {rid(r) for r in load(f"{D}/{f}")} for f in files}
mp = f"{D}/SUBSAMPLE_MANIFEST.json"
if os.path.exists(mp):
kept = json.load(open(mp))["kept_rids_per_behaviour"]
for f in files:
m = set(kept[f.split("__")[0]]); assert m == exp[f], f"subsample manifest disagrees with {f} ({len(m)} vs {len(exp[f])})"
res = {}; ok = True
for f in files:
p = f"{OUT}/{f}"; got = set(); bad = 0; extra = set(); dup = 0
if os.path.exists(p):
for r in load(p):
i = rid(r)
if not str(r.get("model_tag", "")).endswith("__" + SHA): bad += 1; continue
if i in got or i in extra: dup += 1; continue
(got if i in exp[f] else extra).add(i)
miss = exp[f] - got; st = "ok" if not miss and not extra and not bad and not dup else "incomplete"
ok &= st == "ok"; res[f] = {"expected": len(exp[f]), "have": len(got), "missing": len(miss), "extra": len(extra), "badsha": bad, "dup": dup, "status": st}
if not Q: print(f"{f:50s} exp {len(exp[f]):5d} have {len(got):5d} missing {len(miss):5d} extra {len(extra):3d} badsha {bad:4d} dup {dup:3d} {st}")
tot = sum(v["expected"] for v in res.values()); have = sum(v["have"] for v in res.values())
summary = {"complete": ok, "files": len(files), "expected_rows": tot, "rows_ok": have, "sha256": SHA, "out": OUT, "per_file": res}
print(json.dumps(summary if JS else {k: v for k, v in summary.items() if k != "per_file"}))
sys.exit(0 if ok else 1)