Buckets:
| #!/usr/bin/env python3 | |
| """Response-only scoring of the dynamic (per-token top-K) checkpoints, compared to | |
| fixed SAE-50, SAE-150, and raw-hidden. Densifies test records from the sparse GPU cache.""" | |
| from __future__ import annotations | |
| import json, sys | |
| from pathlib import Path | |
| import numpy as np, torch | |
| sys.path.insert(0, "/workspace/scope/modeling") | |
| from models import PAD, TOPICS, build_model, topic_overlap | |
| from sklearn.metrics import f1_score, classification_report, accuracy_score | |
| ROOT = Path("/workspace/scope") | |
| CACHE = ROOT / "cache/dyn150_sparse/sparse.pt" | |
| DEVICE = "cuda" | |
| RESP = 2 | |
| CKPT = { | |
| "GRU": ROOT / "results/dyn150_2k/gru/gru_lr3e4/checkpoint_best.pt", | |
| "Transformer": ROOT / "results/dyn150_2k/transformer/transformer_wide/checkpoint_best.pt", | |
| "ConvNeXt": ROOT / "results/dyn150_2k/tcn/convnext/checkpoint_best.pt", | |
| } | |
| def load_cache(): | |
| d = torch.load(CACHE, map_location="cpu") | |
| off = np.concatenate(([0], np.cumsum(d["lengths"].numpy()))) | |
| return dict(col=d["col"].to(DEVICE), val=d["val"].to(DEVICE), tok_ptr=d["tok_ptr"].to(DEVICE), | |
| labels=d["labels"].to(DEVICE), roles=d["roles"].to(DEVICE), offsets=off, | |
| test=d["test"].numpy(), W=int(d["width"])) | |
| def densify_one(C, r): | |
| lo, hi = int(C["offsets"][r]), int(C["offsets"][r+1]); T = hi - lo | |
| x = torch.zeros(1, T, C["W"], device=DEVICE) | |
| ptr = C["tok_ptr"][lo:hi+1]; nlo, nhi = int(ptr[0]), int(ptr[-1]) | |
| if nhi > nlo: | |
| counts = ptr[1:] - ptr[:-1] | |
| rows = torch.repeat_interleave(torch.arange(T, device=DEVICE), counts) | |
| x[0, rows, C["col"][nlo:nhi]] = C["val"][nlo:nhi].float() | |
| return x, lo, hi | |
| def score(ckpt, C): | |
| p = torch.load(ckpt, map_location="cpu", weights_only=False) | |
| m = build_model(p["architecture"], p["model"]); m.load_state_dict(p["state_dict"]); m.to(DEVICE).eval() | |
| T_all, P_all = [], [] | |
| for r in C["test"]: | |
| r = int(r); x, lo, hi = densify_one(C, r) | |
| pred = m(x)[0].argmax(-1).cpu().numpy() | |
| lab = C["labels"][lo:hi].cpu().numpy().copy(); role = C["roles"][lo:hi].cpu().numpy() | |
| lab[role != RESP] = PAD | |
| keep = lab != PAD | |
| T_all.append(lab[keep]); P_all.append(pred[keep]) | |
| truth = np.concatenate(T_all); predk = np.concatenate(P_all) | |
| ov = topic_overlap(truth, predk) | |
| rep = classification_report(truth, predk, labels=range(len(TOPICS)), target_names=TOPICS, | |
| output_dict=True, zero_division=0) | |
| per = {} | |
| for t in TOPICS: | |
| tm = truth == TOPICS.index(t) | |
| per[t] = {"f1": rep[t]["f1-score"], "iou": ov["per_topic"][t]["iou"], | |
| "acc": float((predk[tm] == TOPICS.index(t)).mean()) if tm.any() else 0.0} | |
| return {"params": sum(x.numel() for x in m.parameters()), | |
| "acc": float(accuracy_score(truth, predk)), | |
| "macro_f1": float(f1_score(truth, predk, average="macro", zero_division=0)), | |
| "macro_iou": ov["macro_iou"], "per_topic": per} | |
| def main(): | |
| C = load_cache() | |
| dyn = {f: score(c, C) for f, c in CKPT.items()} | |
| json.dump(dyn, open(ROOT/"results/dyn150_2k/dynamic_scores.json", "w"), indent=2) | |
| tw = json.load(open(ROOT/"results/sae1500_2k/three_way.json")) # has 50/150/raw | |
| fams = ["GRU", "Transformer", "ConvNeXt"] | |
| print("\n============ OVERALL (response tokens): 50 / 150 / DYNAMIC / raw ============") | |
| for label, key in [("Accuracy","acc"), ("Macro F1 (=Dice)","macro_f1"), ("Macro IoU","macro_iou")]: | |
| print(f"\n -- {label} --") | |
| print(f" {'model':12s} {'SAE-50':>8s} {'SAE-150':>8s} {'DYNAMIC':>8s} {'RAW':>8s} {'dyn-150':>8s}") | |
| for f in fams: | |
| a=tw[f]['50'][key]; b=tw[f]['150'][key]; g=dyn[f][key]; r=tw[f]['raw'][key] | |
| print(f" {f:12s} {a:8.4f} {b:8.4f} {g:8.4f} {r:8.4f} {g-b:+8.4f}") | |
| print(f"\n -- Params (M): 150 vs dynamic --") | |
| for f in fams: | |
| print(f" {f:12s} 150={tw[f]['150']['params']/1e6:6.2f} dynamic={dyn[f]['params']/1e6:6.2f}") | |
| for label, key in [("F1 (=Dice)","f1"), ("IoU","iou"), ("Accuracy","acc")]: | |
| print(f"\n============ PER-TOPIC {label} (response): 150 -> DYNAMIC (Δ), [raw] ============") | |
| print(f"{'topic':22s}" + "".join(f"{f[:10]:>26s}" for f in fams)) | |
| for t in TOPICS: | |
| line = f"{t:22s}" | |
| for f in fams: | |
| b=tw[f]['150']['per_topic'][t][key]; g=dyn[f]['per_topic'][t][key]; r=tw[f]['raw']['per_topic'][t][key] | |
| line += f" {b:.3f}->{g:.3f} {g-b:+.3f}[{r:.3f}]" | |
| print(line) | |
| print("\n============ REGRESSIONS: dynamic vs fixed-150 (Δ < -0.005) ============") | |
| any_reg = False | |
| for f in fams: | |
| for k in ["macro_f1","macro_iou","acc"]: | |
| d = dyn[f][k]-tw[f]['150'][k] | |
| if d < -0.005: print(f" {f:11s} MACRO {k:9s} {tw[f]['150'][k]:.3f} -> {dyn[f][k]:.3f} ({d:+.3f})"); any_reg=True | |
| for t in TOPICS: | |
| for key in ["f1","iou"]: | |
| d = dyn[f]['per_topic'][t][key]-tw[f]['150']['per_topic'][t][key] | |
| if d < -0.005: print(f" {f:11s} {t:22s} {key:3s} {tw[f]['150']['per_topic'][t][key]:.3f} -> {dyn[f]['per_topic'][t][key]:.3f} ({d:+.3f})"); any_reg=True | |
| if not any_reg: print(" none beyond -0.005") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.33 kB
- Xet hash:
- 644e9cd09553dea390659566f90371aa1766c2f820a5aaa5ea619bc94c51f799
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.