Download check_rows.py from jprivera/260926_matched_eval: direct link, hf CLI and curl.
- Browser
- Download file 2.61 kB
-
https://huggingface.co/jprivera/260926_matched_eval/resolve/main/check_rows.py
- Command line
-
hf download hf://jprivera/260926_matched_eval/check_rows.py
-
curl -L -o check_rows.py https://huggingface.co/jprivera/260926_matched_eval/resolve/main/check_rows.py
2.61 kB
| #!/usr/bin/env python3 | |
| """Exact-set completion + provenance check for one eval output dir (used by the worker, 05's requeue and 06_grade.py). | |
| Expected row ids per file come from D/SUBSAMPLE_MANIFEST.json (kept_rids_per_behaviour, cross-checked against D/*.jsonl) or, for a | |
| plain source dir (selection set), from D/*.jsonl directly. A row counts only if its model_tag ends with "__<sha256>" of the adapter | |
| that was supposed to be evaluated. Prints one line per file (ok/missing/extra/badsha/dup) and a summary; exit 0 iff every file has exactly | |
| the expected id set with the right sha, no extra ids and no id twice (a duplicate id = two writers appended = incomplete, AUDIT_live_B 2). | |
| Usage: check_rows.py <D> <OUT_DIR> <sha256> [--json] [--quiet]""" | |
| import json, os, sys, glob | |
| D, OUT, SHA = sys.argv[1:4]; JS = "--json" in sys.argv; Q = "--quiet" in sys.argv | |
| def rid(r): return str(r.get("row_uid", r.get("qid"))) | |
| def load(p): | |
| for l in open(p): | |
| if l.strip(): | |
| try: yield json.loads(l) | |
| except Exception: pass | |
| files = sorted(os.path.basename(f) for f in glob.glob(f"{D}/*.jsonl")) | |
| assert files, f"no *.jsonl under {D}" | |
| exp = {f: {rid(r) for r in load(f"{D}/{f}")} for f in files} | |
| mp = f"{D}/SUBSAMPLE_MANIFEST.json" | |
| if os.path.exists(mp): | |
| kept = json.load(open(mp))["kept_rids_per_behaviour"] | |
| for f in files: | |
| m = set(kept[f.split("__")[0]]); assert m == exp[f], f"subsample manifest disagrees with {f} ({len(m)} vs {len(exp[f])})" | |
| res = {}; ok = True | |
| for f in files: | |
| p = f"{OUT}/{f}"; got = set(); bad = 0; extra = set(); dup = 0 | |
| if os.path.exists(p): | |
| for r in load(p): | |
| i = rid(r) | |
| if not str(r.get("model_tag", "")).endswith("__" + SHA): bad += 1; continue | |
| if i in got or i in extra: dup += 1; continue | |
| (got if i in exp[f] else extra).add(i) | |
| miss = exp[f] - got; st = "ok" if not miss and not extra and not bad and not dup else "incomplete" | |
| ok &= st == "ok"; res[f] = {"expected": len(exp[f]), "have": len(got), "missing": len(miss), "extra": len(extra), "badsha": bad, "dup": dup, "status": st} | |
| if not Q: print(f"{f:50s} exp {len(exp[f]):5d} have {len(got):5d} missing {len(miss):5d} extra {len(extra):3d} badsha {bad:4d} dup {dup:3d} {st}") | |
| tot = sum(v["expected"] for v in res.values()); have = sum(v["have"] for v in res.values()) | |
| summary = {"complete": ok, "files": len(files), "expected_rows": tot, "rows_ok": have, "sha256": SHA, "out": OUT, "per_file": res} | |
| print(json.dumps(summary if JS else {k: v for k, v in summary.items() if k != "per_file"})) | |
| sys.exit(0 if ok else 1) | |