Download assert_arms.py from jprivera/260926_matched_eval: direct link, hf CLI and curl.
- Browser
- Download file 3.04 kB
-
https://huggingface.co/jprivera/260926_matched_eval/resolve/main/assert_arms.py
- Command line
-
hf download hf://jprivera/260926_matched_eval/assert_arms.py
-
curl -L -o assert_arms.py https://huggingface.co/jprivera/260926_matched_eval/resolve/main/assert_arms.py
3.04 kB
| """PAIRED GRID assertions before eval launch (driver stage B). Exit 0 only if all hold: | |
| 96 arms = 2 fams x 3 organisms x (5 methods x 3 doses + 1 untreated); every sft arm from runs_orig/, every cws arm from cws/, every | |
| dpo/npo/rret arm from runs/; every arm name <fam>_o<o>_...; the 6 untreated arms point at the pinned organism adapters (sha == config.sh | |
| pin); manifests/VERIFY_OK newer than arms_manifest.json; every CWS pack's P0 sha == its organism's pin and its branches come from | |
| runs_orig/<fam>/o<o>/.""" | |
| import json, os, sys, subprocess | |
| E = os.path.dirname(os.path.abspath(__file__)); ok = True | |
| C = dict(l.split("=", 1) for l in subprocess.run(["bash", "-c", f"source {E}/config.sh; for v in $(compgen -v); do printf '%s=%s\\n' \"$v\" \"${{!v}}\"; done"], capture_output=True, text=True).stdout.splitlines() if "=" in l and not any(x in l.split("=")[0].upper() for x in ("TOKEN", "KEY", "SECRET", "PASS", "WANDB", "GITHUB"))) | |
| def bad(m): | |
| global ok; ok = False; print("ASSERT FAIL:", m) | |
| if not os.path.exists(f"{E}/arms/arms_fixed_dose.txt"): print("ASSERT FAILED: no arms/arms_fixed_dose.txt (run 04_build_arms.py first)"); sys.exit(1) | |
| arms = [l.strip().split("|") for l in open(f"{E}/arms/arms_fixed_dose.txt") if l.strip()] | |
| if len(arms) != 96: bad(f"{len(arms)} arms (need 96 = 6 untreated + 90 treated)") | |
| for n, fam, d, sha in arms: | |
| p = n.split("_") | |
| if p[0] != fam or p[1] not in ("o0", "o1", "o2"): bad(f"{n}: name/family field mismatch ({fam})") | |
| if p[2] == "untreated": | |
| pin = C.get(f"{fam.upper()}_ADAPTER_{p[1]}_SHA"); pad = C.get(f"{fam.upper()}_ADAPTER_{p[1]}") | |
| if sha != pin: bad(f"{n}: sha {sha[:12]} != pinned organism adapter sha {str(pin)[:12]}") | |
| if d != pad: bad(f"{n}: adapter_dir {d} != pinned organism adapter dir {pad}") | |
| continue | |
| m = p[2] | |
| if m == "sft" and f"/runs_orig/{fam}/{p[1]}/" not in d: bad(f"{n}: SFT arm not from runs_orig/{fam}/{p[1]}: {d}") | |
| if m in ("dpo", "npo", "rret") and f"/runs/{fam}/{p[1]}/" not in d: bad(f"{n}: {m} arm not from runs/{fam}/{p[1]}: {d}") | |
| if m == "cws" and f"/cws/{fam}_{p[1]}_cws_" not in d: bad(f"{n}: cws arm dir {d}") | |
| if m not in ("sft", "dpo", "npo", "cws", "rret"): bad(f"{n}: unknown method {m}") | |
| pm = json.load(open(f"{E}/cws/PACK_MANIFEST.json")) if os.path.exists(f"{E}/cws/PACK_MANIFEST.json") else {} | |
| for k, v in pm.items(): | |
| fam, o = k.split("_")[0], k.split("_")[1] | |
| if v.get("p0_sha256") != C.get(f"{fam.upper()}_ADAPTER_{o}_SHA"): bad(f"CWS {k}: P0 sha {str(v.get('p0_sha256'))[:12]} != pinned {fam} {o} organism adapter") | |
| for key in ("pos", "neg"): | |
| if f"/runs_orig/{fam}/{o}/" not in str(v.get(key)): bad(f"CWS {k} {key} not from runs_orig/{fam}/{o}: {v.get(key)}") | |
| st = f"{E}/manifests/VERIFY_OK"; am = f"{E}/manifests/arms_manifest.json" | |
| if not os.path.exists(st) or not os.path.exists(am) or os.path.getmtime(st) < os.path.getmtime(am): bad("VERIFY_OK missing or older than arms_manifest.json") | |
| print("ASSERT OK" if ok else "ASSERT FAILED"); sys.exit(0 if ok else 1) | |