Buckets:
| #!/usr/bin/env python3 | |
| """Prompt vs response vs combined macro-F1/IoU for each family winner, 1B and 4B.""" | |
| import sys | |
| sys.path.insert(0, "/workspace/local/modeling") # models.py (post-reorg) | |
| sys.path.insert(0, "/workspace/local/scrap/scripts") # extra_metrics.py (post-reorg) | |
| import numpy as np, torch | |
| from pathlib import Path | |
| from models import build_model | |
| import extra_metrics as em | |
| R = "/workspace/local/" | |
| C1 = R + "cache/sae500_2k_clean_prompt_response" | |
| C4 = R + "cache/sae500_2k_gemma4b" | |
| ROWS = [ | |
| ("1B", "GRU", R + "results/gru_2k/gru_lr3e4/checkpoint_best.pt", C1), | |
| ("1B", "Transformer", R + "results/transformer_2k/transformer_wide/checkpoint_best.pt", C1), | |
| ("1B", "ConvNeXt TCN", R + "results/convnext_windows/window_505/checkpoint_best.pt", C1), | |
| ("4B", "GRU", R + "results/gemma4b_ablations/gru_2k__gru_wide/gru_wide/checkpoint_best.pt", C4), | |
| ("4B", "Transformer", R + "results/gemma4b_ablations/transformer_2k__transformer_deeper/transformer_deeper/checkpoint_best.pt", C4), | |
| ("4B", "ConvNeXt TCN", R + "results/gemma4b_baselines/convnext_window505/checkpoint_best.pt", C4), | |
| ] | |
| def scope(ck, cache): | |
| p = torch.load(ck, map_location="cpu", weights_only=False) | |
| m = build_model(p["architecture"], p["model"]); m.load_state_dict(p["state_dict"]); m.eval() | |
| c = Path(cache); starts = em.offsets(np.load(c / "lengths.npy")) | |
| feats = np.load(c / "features.npy", mmap_mode="r"); labs = np.load(c / "labels.npy", mmap_mode="r") | |
| rls = np.load(c / "role_ids.npy", mmap_mode="r"); test = np.load(c / "split_indices.npz")["test"] | |
| P, T, Rl = [], [], [] | |
| with torch.inference_mode(): | |
| for idx in test: | |
| lo, hi = int(starts[idx]), int(starts[idx + 1]) | |
| x = torch.from_numpy(np.asarray(feats[lo:hi], dtype=np.float32))[None] | |
| P.append(m(x)[0].argmax(-1).numpy()); T.append(np.asarray(labs[lo:hi])); Rl.append(np.asarray(rls[lo:hi])) | |
| P, T, Rl = np.concatenate(P), np.concatenate(T), np.concatenate(Rl) | |
| return em.macro_overlap(P, T), em.macro_overlap(P[Rl == 1], T[Rl == 1]), em.macro_overlap(P[Rl == 2], T[Rl == 2]) | |
| print(f"{'BB':3}{'family':14}{'promptF1':>9}{'respF1':>8}{'combF1':>8}{'resp-prompt':>12}{'promptIoU':>10}{'respIoU':>9}") | |
| for bb, fam, ck, cache in ROWS: | |
| comb, pr, rs = scope(ck, cache) | |
| print(f"{bb:3}{fam:14}{pr['macro_dice']*100:9.2f}{rs['macro_dice']*100:8.2f}{comb['macro_dice']*100:8.2f}" | |
| f"{(rs['macro_dice']-pr['macro_dice'])*100:12.2f}{pr['macro_iou']*100:10.2f}{rs['macro_iou']*100:9.2f}") | |
Xet Storage Details
- Size:
- 2.55 kB
- Xet hash:
- b81a61446efd659b734c37ad8d1751206e2a31fcd9c4f92fd2ea59a3331095cf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.