260926_matched_eval / assert_arms.py
jprivera's picture
upload batch 1/7 from pod 38.80.152.76
43eb135 verified
Raw History Blame Contribute Delete
3.04 kB
"""PAIRED GRID assertions before eval launch (driver stage B). Exit 0 only if all hold:
96 arms = 2 fams x 3 organisms x (5 methods x 3 doses + 1 untreated); every sft arm from runs_orig/, every cws arm from cws/, every
dpo/npo/rret arm from runs/; every arm name <fam>_o<o>_...; the 6 untreated arms point at the pinned organism adapters (sha == config.sh
pin); manifests/VERIFY_OK newer than arms_manifest.json; every CWS pack's P0 sha == its organism's pin and its branches come from
runs_orig/<fam>/o<o>/."""
import json, os, sys, subprocess
E = os.path.dirname(os.path.abspath(__file__)); ok = True
C = dict(l.split("=", 1) for l in subprocess.run(["bash", "-c", f"source {E}/config.sh; for v in $(compgen -v); do printf '%s=%s\\n' \"$v\" \"${{!v}}\"; done"], capture_output=True, text=True).stdout.splitlines() if "=" in l and not any(x in l.split("=")[0].upper() for x in ("TOKEN", "KEY", "SECRET", "PASS", "WANDB", "GITHUB")))
def bad(m):
global ok; ok = False; print("ASSERT FAIL:", m)
if not os.path.exists(f"{E}/arms/arms_fixed_dose.txt"): print("ASSERT FAILED: no arms/arms_fixed_dose.txt (run 04_build_arms.py first)"); sys.exit(1)
arms = [l.strip().split("|") for l in open(f"{E}/arms/arms_fixed_dose.txt") if l.strip()]
if len(arms) != 96: bad(f"{len(arms)} arms (need 96 = 6 untreated + 90 treated)")
for n, fam, d, sha in arms:
p = n.split("_")
if p[0] != fam or p[1] not in ("o0", "o1", "o2"): bad(f"{n}: name/family field mismatch ({fam})")
if p[2] == "untreated":
pin = C.get(f"{fam.upper()}_ADAPTER_{p[1]}_SHA"); pad = C.get(f"{fam.upper()}_ADAPTER_{p[1]}")
if sha != pin: bad(f"{n}: sha {sha[:12]} != pinned organism adapter sha {str(pin)[:12]}")
if d != pad: bad(f"{n}: adapter_dir {d} != pinned organism adapter dir {pad}")
continue
m = p[2]
if m == "sft" and f"/runs_orig/{fam}/{p[1]}/" not in d: bad(f"{n}: SFT arm not from runs_orig/{fam}/{p[1]}: {d}")
if m in ("dpo", "npo", "rret") and f"/runs/{fam}/{p[1]}/" not in d: bad(f"{n}: {m} arm not from runs/{fam}/{p[1]}: {d}")
if m == "cws" and f"/cws/{fam}_{p[1]}_cws_" not in d: bad(f"{n}: cws arm dir {d}")
if m not in ("sft", "dpo", "npo", "cws", "rret"): bad(f"{n}: unknown method {m}")
pm = json.load(open(f"{E}/cws/PACK_MANIFEST.json")) if os.path.exists(f"{E}/cws/PACK_MANIFEST.json") else {}
for k, v in pm.items():
fam, o = k.split("_")[0], k.split("_")[1]
if v.get("p0_sha256") != C.get(f"{fam.upper()}_ADAPTER_{o}_SHA"): bad(f"CWS {k}: P0 sha {str(v.get('p0_sha256'))[:12]} != pinned {fam} {o} organism adapter")
for key in ("pos", "neg"):
if f"/runs_orig/{fam}/{o}/" not in str(v.get(key)): bad(f"CWS {k} {key} not from runs_orig/{fam}/{o}: {v.get(key)}")
st = f"{E}/manifests/VERIFY_OK"; am = f"{E}/manifests/arms_manifest.json"
if not os.path.exists(st) or not os.path.exists(am) or os.path.getmtime(st) < os.path.getmtime(am): bad("VERIFY_OK missing or older than arms_manifest.json")
print("ASSERT OK" if ok else "ASSERT FAILED"); sys.exit(0 if ok else 1)