Add the verified unconditional control checkpoint (protein descriptors provably decorrelated from recipes in training data) that resolves the earlier unexplained ablation anomaly -- it correctly stays at chance (p=0.54-0.81) on the identity-fixed specificity test.
9dd51de verified Download src/train_unconditional_control.py from Sravankumarbonthada/bioform-lm: direct link, hf CLI and curl.
- Browser
- Download file 8.6 kB
-
https://huggingface.co/Sravankumarbonthada/bioform-lm/resolve/main/src/train_unconditional_control.py
- Command line
-
hf download hf://Sravankumarbonthada/bioform-lm/src/train_unconditional_control.py
-
curl -L -o train_unconditional_control.py https://huggingface.co/Sravankumarbonthada/bioform-lm/resolve/main/src/train_unconditional_control.py
8.6 kB
| #!/usr/bin/env python3 | |
| """ | |
| Train a properly documented, auditable "unconditional" control model. | |
| The earlier checkpoint labelled `checkpoints_unconditional` reached statistical | |
| significance on the identity-fixed protein-specificity test (Section on the | |
| identity-merging bug fix) for reasons we could not fully re-verify -- its | |
| training provenance was uncertain. That made it useless as a negative control: | |
| if we cannot rule out that it secretly saw protein-recipe structure during | |
| training, its passing the specificity test proves nothing either way. | |
| This script builds a clean one instead. It takes the EXACT SAME corpus used to | |
| train `checkpoints_conditional` (data/conditional_training_data.parquet) and | |
| independently permutes the three protein-descriptor columns | |
| (protein_mw_kda, protein_pi, protein_tm_baseline_c) as a block, row-for-row, | |
| so that: | |
| * the marginal distribution of protein descriptors is unchanged (same values, | |
| same frequency), | |
| * the marginal distribution of recipes and stability scores is unchanged | |
| (same rows, same is_preferred flags, same buffer/pH/etc.), | |
| * but the PAIRING between a given protein descriptor and the recipe/outcome | |
| it appears next to is now uniformly random. | |
| P(recipe | protein) is therefore provably uninformative in this corpus by | |
| construction: whatever correlation existed between a protein's descriptor and | |
| its simulator-preferred recipe has been destroyed. A model trained on this | |
| corpus cannot learn genuine protein-conditional structure -- it can at most | |
| learn to attend to the descriptor tokens as decoration. This is the clean | |
| negative control the earlier "unconditional" checkpoint should have been. | |
| Everything else (architecture, loss, optimizer, epochs, batch size, seed) is | |
| held identical to scripts/train_generative.py so the comparison isolates | |
| exactly one variable: whether protein and recipe are correlated in training. | |
| python scripts/train_unconditional_control.py --epochs 30 --batch-size 4096 | |
| """ | |
| import argparse | |
| import json | |
| import logging | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from model.bioform_lm import BioFormLM, PREFIX_LEN, RECIPE_LEN | |
| from scripts.train_generative import encode_parallel, lm_loss | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s", datefmt="%H:%M:%S") | |
| log = logging.getLogger("train_unconditional") | |
| SEQ_LEN = PREFIX_LEN + RECIPE_LEN | |
| DESCRIPTOR_COLS = ["protein_mw_kda", "protein_pi", "protein_tm_baseline_c"] | |
| def decorrelate(df: pd.DataFrame, seed: int) -> pd.DataFrame: | |
| """Independently permute the descriptor block, breaking protein<->recipe pairing.""" | |
| rng = np.random.default_rng(seed) | |
| perm = rng.permutation(len(df)) | |
| out = df.copy().reset_index(drop=True) | |
| shuffled = df[DESCRIPTOR_COLS].reset_index(drop=True).iloc[perm].reset_index(drop=True) | |
| out[DESCRIPTOR_COLS] = shuffled | |
| return out | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--data", default="data/conditional_training_data.parquet") | |
| ap.add_argument("--epochs", type=int, default=30) | |
| ap.add_argument("--batch-size", type=int, default=4096) | |
| ap.add_argument("--lr", type=float, default=1e-3) | |
| ap.add_argument("--lm-weight", type=float, default=1.0) | |
| ap.add_argument("--val-frac", type=float, default=0.1) | |
| ap.add_argument("--workers", type=int, default=24) | |
| ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") | |
| ap.add_argument("--out", default="experiments/checkpoints_unconditional_v2") | |
| ap.add_argument("--seed", type=int, default=42) | |
| ap.add_argument("--decorrelate-seed", type=int, default=777) | |
| a = ap.parse_args() | |
| torch.manual_seed(a.seed) | |
| np.random.seed(a.seed) | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| torch.backends.cudnn.allow_tf32 = True | |
| out = Path(a.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| df = pd.read_parquet(a.data) | |
| df = decorrelate(df, a.decorrelate_seed) | |
| log.info(f"decorrelated {len(df):,} rows: protein descriptors permuted " | |
| f"independently of recipe/outcome (seed={a.decorrelate_seed})") | |
| # sanity check: descriptor-recipe pairing must now be different from the original | |
| orig = pd.read_parquet(a.data) | |
| same_pairing = (orig[DESCRIPTOR_COLS].values == df[DESCRIPTOR_COLS].values).all(axis=1).mean() | |
| log.info(f"fraction of rows where descriptor accidentally landed back on its own " | |
| f"recipe: {same_pairing:.5f} (expect ~0)") | |
| assert same_pairing < 0.01, "decorrelation failed -- pairing not broken" | |
| t0 = time.time() | |
| seqs, bins, pref = encode_parallel(df, a.workers) | |
| log.info(f"encoded {len(seqs):,} sequences (len {seqs.shape[1]}) " | |
| f"on {a.workers} workers in {time.time()-t0:.0f}s") | |
| g = torch.Generator().manual_seed(a.seed) | |
| perm = torch.randperm(len(seqs), generator=g) | |
| n_val = int(len(seqs) * a.val_frac) | |
| val_i, tr_i = perm[:n_val], perm[n_val:] | |
| dev = a.device | |
| Xtr, Ytr, Wtr = seqs[tr_i].to(dev), bins[tr_i].to(dev), pref[tr_i].to(dev) | |
| Xva, Yva, Wva = seqs[val_i].to(dev), bins[val_i].to(dev), pref[val_i].to(dev) | |
| from model.tokenizer import FormulationTokenizer | |
| vocab_size = FormulationTokenizer().vocab_size | |
| model = BioFormLM(vocab_size=vocab_size).to(dev) | |
| log.info(f"model {sum(p.numel() for p in model.parameters()):,} params | " | |
| f"train {len(Xtr):,} val {len(Xva):,} | batch {a.batch_size}") | |
| opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.01) | |
| steps = max(1, len(Xtr) // a.batch_size) | |
| sched = torch.optim.lr_scheduler.OneCycleLR( | |
| opt, max_lr=a.lr, total_steps=a.epochs * steps, pct_start=0.1) | |
| use_amp = dev == "cuda" | |
| scaler = torch.cuda.amp.GradScaler(enabled=use_amp) | |
| def run_eval(): | |
| model.eval() | |
| tot = np.zeros(3); correct = n = 0 | |
| with torch.no_grad(): | |
| for i in range(0, len(Xva), a.batch_size): | |
| x, y, w = Xva[i:i+a.batch_size], Yva[i:i+a.batch_size], Wva[i:i+a.batch_size] | |
| with torch.autocast("cuda", dtype=torch.bfloat16, enabled=use_amp): | |
| s, r = model(x) | |
| ls = F.cross_entropy(s.float(), y); ll = lm_loss(r, x, w) | |
| bs = len(x) | |
| tot += np.array([(ls + a.lm_weight*ll).item(), ls.item(), ll.item()]) * bs | |
| correct += (s.argmax(-1) == y).sum().item(); n += bs | |
| return tot / n, correct / n | |
| best, hist = float("inf"), [] | |
| for ep in range(1, a.epochs + 1): | |
| model.train(); t = time.time() | |
| idx = torch.randperm(len(Xtr), device=dev) | |
| agg = np.zeros(3); seen = 0 | |
| for i in range(0, len(idx) - a.batch_size + 1, a.batch_size): | |
| b = idx[i:i+a.batch_size] | |
| x, y, w = Xtr[b], Ytr[b], Wtr[b] | |
| with torch.autocast("cuda", dtype=torch.bfloat16, enabled=use_amp): | |
| s, r = model(x) | |
| ls = F.cross_entropy(s.float(), y) | |
| ll = lm_loss(r, x, w) | |
| loss = ls + a.lm_weight * ll | |
| opt.zero_grad(set_to_none=True) | |
| scaler.scale(loss).backward() | |
| scaler.unscale_(opt) | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| scaler.step(opt); scaler.update(); sched.step() | |
| agg += np.array([loss.item(), ls.item(), ll.item()]) * len(b); seen += len(b) | |
| agg /= max(seen, 1) | |
| v, acc = run_eval() | |
| log.info(f"epoch {ep:>3}/{a.epochs} train {agg[0]:.4f} | val {v[0]:.4f} " | |
| f"acc {acc:.3f} ppl {np.exp(v[2]):.3f} [{time.time()-t:.1f}s]") | |
| hist.append({"epoch": ep, "train": agg.tolist(), "val": v.tolist(), | |
| "stab_acc": acc, "recipe_ppl": float(np.exp(v[2]))}) | |
| if v[0] < best: | |
| best = v[0] | |
| torch.save({"epoch": ep, "model_state_dict": model.state_dict(), | |
| "val_loss": v[0], "vocab_size": vocab_size, | |
| "decorrelate_seed": a.decorrelate_seed, | |
| "note": "protein descriptors independently permuted vs " | |
| "recipe/outcome; P(recipe|protein) destroyed by " | |
| "construction -- see script docstring"}, | |
| out / "best.pt") | |
| (out / "history.json").write_text(json.dumps(hist, indent=2)) | |
| log.info(f"done. best val {best:.4f} -> {out/'best.pt'}") | |
| if __name__ == "__main__": | |
| main() | |