Download code/ampd/ae_latent_selection.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 13 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/ae_latent_selection.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/ae_latent_selection.py
-
curl -L -o ae_latent_selection.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/ae_latent_selection.py
13 kB
| """Protein autoencoder, patient-level latent screening, and regression heads.""" | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import warnings | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from scipy.stats import false_discovery_control, rankdata, t | |
| from sklearn.linear_model import Ridge | |
| LATENT = 32 | |
| MONTHS = np.array([0, 6, 12, 18, 24, 36, 48, 60, 72, 84]) | |
| ARMS = ("month", "missingness", "ae_all", "ae_topk", "ae_fdr", "ae_topk_month") | |
| ALPHAS = (1., 10., 100., 1000.) | |
| TOP_K = (2, 4, 8, 16) | |
| FACTORS = (0., .5, .75, 1.) | |
| class ProteinAutoencoder(torch.nn.Module): | |
| def __init__(self, n_proteins, latent=LATENT): | |
| super().__init__() | |
| self.encoder = torch.nn.Sequential(torch.nn.Linear(n_proteins, 128), torch.nn.ReLU(), | |
| torch.nn.Linear(128, 64), torch.nn.ReLU(), torch.nn.Linear(64, latent)) | |
| self.decoder = torch.nn.Sequential(torch.nn.Linear(latent, 64), torch.nn.ReLU(), | |
| torch.nn.Linear(64, 128), torch.nn.ReLU(), torch.nn.Linear(128, n_proteins)) | |
| def forward(self, x): | |
| return self.decoder(self.encoder(x)) | |
| def fit_preprocessor(proteins): | |
| values = np.asarray(proteins, dtype=float) | |
| log = np.where(np.isfinite(values), np.log1p(values), np.nan) | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore", RuntimeWarning) | |
| impute = np.nan_to_num(np.nanmedian(log, axis=0)) | |
| filled = np.where(np.isfinite(log), log, impute) | |
| mean, scale = filled.mean(0), filled.std(0) | |
| return {"impute": impute, "mean": mean, "scale": np.where(scale > 1e-12, scale, 1.)} | |
| def transform_proteins(proteins, params): | |
| values = np.asarray(proteins, dtype=float) | |
| if np.any(values < 0): | |
| raise ValueError("Protein abundance must be nonnegative") | |
| observed = np.isfinite(values) | |
| log = np.log1p(values) | |
| filled = np.where(observed, log, params["impute"]) | |
| return np.clip((filled-params["mean"])/params["scale"], -5., 5.), observed | |
| def masked_reconstruction_loss(prediction, target, observed): | |
| return (((prediction-target)**2)*observed).sum()/observed.sum().clamp_min(1.) | |
| def train_autoencoder(x, mask, seed, epochs, validation=None): | |
| torch.manual_seed(seed) | |
| model = ProteinAutoencoder(x.shape[1]) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=.001, weight_decay=.0001) | |
| xt, mt = torch.tensor(x, dtype=torch.float32), torch.tensor(mask, dtype=torch.float32) | |
| rng = np.random.default_rng(seed) | |
| best, chosen, stale, best_state = float("inf"), epochs, 0, None | |
| logs = [] | |
| if validation is not None: | |
| vx, vm = [torch.tensor(v, dtype=torch.float32) for v in validation] | |
| for epoch in range(1, epochs+1): | |
| model.train() | |
| total, count = 0., 0. | |
| permutation = rng.permutation(len(x)) | |
| for start in range(0, len(x), 64): | |
| rows = permutation[start:start+64] | |
| output = model(xt[rows]) | |
| loss = masked_reconstruction_loss(output, xt[rows], mt[rows]) | |
| if not torch.isfinite(loss): | |
| raise ValueError("Nonfinite reconstruction loss") | |
| optimizer.zero_grad() | |
| loss.backward() | |
| optimizer.step() | |
| n = float(mt[rows].sum()) | |
| total += float(loss.detach())*n | |
| count += n | |
| record = {"epoch": epoch, "train_observed_mse": total/count} | |
| if validation is not None: | |
| model.eval() | |
| with torch.no_grad(): | |
| value = float(masked_reconstruction_loss(model(vx), vx, vm)) | |
| record["validation_observed_mse"] = value | |
| if value < best-1e-5: | |
| best, chosen, stale = value, epoch, 0 | |
| best_state = {k: v.detach().clone() for k, v in model.state_dict().items()} | |
| else: | |
| stale += 1 | |
| logs.append(record) | |
| if validation is not None and stale >= 30: | |
| break | |
| if best_state is not None: | |
| model.load_state_dict(best_state) | |
| return model.eval(), pd.DataFrame(logs), chosen | |
| def export_numeric(model): | |
| return {key.replace(".", "_"): value.detach().numpy().astype(float).copy() | |
| for key, value in model.state_dict().items()} | |
| def numeric_network(x, params, prefix): | |
| result = np.asarray(x, dtype=float) | |
| for layer in [0, 2, 4]: | |
| result = result @ params[f"{prefix}_{layer}_weight"].T + params[f"{prefix}_{layer}_bias"] | |
| if layer != 4: | |
| result = np.maximum(result, 0) | |
| return result | |
| def encode(proteins, preprocessor, network): | |
| x, _ = transform_proteins(proteins, preprocessor) | |
| return numeric_network(x, network, "encoder") | |
| def reconstruction_metrics(proteins, preprocessor, network): | |
| x, mask = transform_proteins(proteins, preprocessor) | |
| reconstructed = numeric_network(numeric_network(x, network, "encoder"), network, "decoder") | |
| error = (reconstructed-x)[mask] | |
| return {"n_samples": len(x), "n_observed_values": int(mask.sum()), | |
| "observed_mse": float(np.mean(error**2)), "observed_mae": float(np.mean(np.abs(error))), | |
| "training_mean_baseline_mse": float(np.mean(x[mask]**2))} | |
| def errors(y, prediction): | |
| return 200*np.abs(y-prediction)/(np.abs(y)+np.abs(prediction)+2) | |
| def centered_ranks(x): | |
| r = rankdata(x, axis=0, method="average") | |
| r -= r.mean(0) | |
| norm = np.sqrt((r*r).sum(0)) | |
| return np.divide(r, norm, out=np.zeros_like(r), where=norm > 0) | |
| def partial_rank(x, y, month, detected): | |
| rx, ry = rankdata(x), rankdata(y) | |
| tm, dc = rankdata(month), rankdata(detected) | |
| tm = (tm-tm.mean())/max(tm.std(), 1.) | |
| dc = (dc-dc.mean())/max(dc.std(), 1.) | |
| z = np.c_[np.ones(len(y)), tm, tm**2, dc] | |
| u, s, _ = np.linalg.svd(z, full_matrices=False) | |
| q = u[:, s > s[0]*max(z.shape)*np.finfo(float).eps] | |
| a, b = rx-q@(q.T@rx), ry-q@(q.T@ry) | |
| xx, yy = a@a, b@b | |
| if xx < 1e-12 or yy < 1e-12: | |
| return np.nan, np.nan | |
| coefficient = (a@b)/xx | |
| residual = b-coefficient*a | |
| leverage = (q*q).sum(1)+a*a/xx | |
| if np.max(leverage) >= 1-1e-10: | |
| return float((a@b)/np.sqrt(xx*yy)), np.nan | |
| variance = np.sum((a*residual/(1-leverage))**2)/xx**2 | |
| p = 2*t.sf(abs(coefficient)/np.sqrt(max(variance, 1e-30)), len(y)-q.shape[1]-1) | |
| return float((a@b)/np.sqrt(xx*yy)), float(p) | |
| def screen_latents(z, samples, y, permutations=99999, seed=42): | |
| """One earliest eligible visit per patient/target; never pools repeated rows.""" | |
| records = [] | |
| for target in range(4): | |
| meta = samples.copy().reset_index(drop=True) | |
| meta["row"] = np.arange(len(meta)) | |
| meta["outcome"] = y[:, target*4] | |
| first = meta.loc[meta.outcome.notna()].sort_values(["uid", "visit_month"]).drop_duplicates("uid") | |
| index = first.row.to_numpy(int) | |
| values = first.outcome.to_numpy(float) | |
| a, b = centered_ranks(z[index]), centered_ranks(values) | |
| rho = a.T @ b | |
| count = np.zeros(z.shape[1], dtype=np.int64) | |
| rng = np.random.default_rng(seed+target) | |
| for start in range(0, permutations, 2048): | |
| size = min(2048, permutations-start) | |
| shuffled = rng.permuted(np.broadcast_to(b, (size, len(b))), axis=1) | |
| count += (np.abs(shuffled@a) >= np.abs(rho)[None, :]-1e-12).sum(0) | |
| p = (count+1)/(permutations+1) | |
| for coordinate in range(z.shape[1]): | |
| active = np.std(z[index, coordinate]) > 1e-10 and np.unique(values).size >= 3 and len(index) >= 30 | |
| r_adjusted, p_adjusted = partial_rank(z[index, coordinate], values, | |
| first.visit_month.to_numpy(), first.detected_count.to_numpy()) if active else (np.nan, np.nan) | |
| records.append({"target": f"updrs_{target+1}", "latent": coordinate, "n_patients": len(index), | |
| "rho": float(rho[coordinate]) if active else np.nan, "p_permutation": float(p[coordinate]) if active else np.nan, | |
| "rho_adjusted": r_adjusted, "p_adjusted": p_adjusted, | |
| "rho_detected_count": float(centered_ranks(z[index, coordinate]) @ centered_ranks(first.detected_count.to_numpy())), | |
| "rho_month": float(centered_ranks(z[index, coordinate]) @ centered_ranks(first.visit_month.to_numpy()))}) | |
| result = pd.DataFrame(records) | |
| for pcol, qcol in [("p_permutation", "q_bh"), ("p_adjusted", "q_adjusted_bh")]: | |
| valid = np.isfinite(result[pcol]) | |
| result[qcol] = np.where(valid, false_discovery_control(np.where(valid, result[pcol], 1.), method="bh"), np.nan) | |
| return result | |
| def selection_mask(correlations, policy, k=0): | |
| mask = np.zeros((4, LATENT), dtype=bool) | |
| for target in range(4): | |
| rows = correlations.loc[correlations.target == f"updrs_{target+1}"].copy() | |
| rows["absolute"] = rows.rho.abs() | |
| if policy == "all": | |
| selected = np.arange(LATENT) | |
| elif policy == "fdr": | |
| selected = rows.loc[(rows.q_bh < .05) & (rows.absolute >= .2), "latent"].to_numpy(int) | |
| elif policy == "topk": | |
| selected = rows.loc[rows.rho.notna()].sort_values(["absolute", "latent"], ascending=[False, True]).head(k).latent.to_numpy(int) | |
| else: | |
| selected = np.empty(0, dtype=int) | |
| mask[target, selected] = True | |
| return mask | |
| def month_features(month): | |
| month = np.asarray(month, dtype=float) | |
| return np.c_[month/84., (month/84.)**2, month[:, None] == MONTHS] | |
| def quality_features(proteins): | |
| missing = ~np.isfinite(proteins) | |
| return np.c_[missing, (~missing).sum(1)].astype(float) | |
| def fit_regression_pre(months, proteins, latent): | |
| result = {} | |
| for name, values in [("month", month_features(months)), ("quality", quality_features(proteins)), ("latent", latent)]: | |
| mean, std = values.mean(0), values.std(0) | |
| result[f"{name}_mean"], result[f"{name}_scale"] = mean, np.where(std > 1e-12, std, 1.) | |
| return result | |
| def regression_features(months, proteins, latent, arm, pre): | |
| if arm == "month": | |
| return (month_features(months)-pre["month_mean"])/pre["month_scale"] | |
| if arm == "missingness": | |
| return (quality_features(proteins)-pre["quality_mean"])/pre["quality_scale"] | |
| z = (latent-pre["latent_mean"])/pre["latent_scale"] | |
| if arm == "ae_topk_month": | |
| m = (month_features(months)-pre["month_mean"])/pre["month_scale"] | |
| return np.c_[m, z] | |
| return z | |
| def fit_heads(x, y, arm, mask, alpha): | |
| coefficients, intercepts = [], [] | |
| for head in range(16): | |
| active = np.ones(x.shape[1], dtype=bool) | |
| if arm.startswith("ae_"): | |
| active[-LATENT:] = mask[head//4] | |
| valid = np.isfinite(y[:, head]) | |
| model = Ridge(alpha=alpha, solver="cholesky").fit(x[valid]*active, y[valid, head]) | |
| coefficients.append(model.coef_*active) | |
| intercepts.append(model.intercept_) | |
| return np.stack(coefficients), np.asarray(intercepts) | |
| def predict_heads(x, coef, intercept, multipliers): | |
| return np.clip(x@coef.T+intercept, 0., 86.) * np.repeat(multipliers, 4) | |
| def calibrate(y, prediction): | |
| multipliers = [] | |
| for target in range(4): | |
| cols = slice(target*4, (target+1)*4) | |
| scores = [float(np.nanmean(errors(y[:, cols], prediction[:, cols]*factor))) for factor in FACTORS] | |
| multipliers.append(FACTORS[int(np.argmin(scores))]) | |
| return multipliers, float(np.nanmean(errors(y, prediction*np.repeat(multipliers, 4)))) | |
| class AELatentPredictor: | |
| def __init__(self, bundle): | |
| bundle = Path(bundle) | |
| self.spec = json.loads((bundle/"model.json").read_text()) | |
| self.pre = dict(np.load(bundle/"protein_preprocessor.npz", allow_pickle=False)) | |
| self.network = dict(np.load(bundle/"autoencoder_numeric.npz", allow_pickle=False)) | |
| self.reg_pre = dict(np.load(bundle/"regression_preprocessor.npz", allow_pickle=False)) | |
| self.heads = {arm: dict(np.load(bundle/f"{arm}.npz", allow_pickle=False)) for arm in ARMS} | |
| def encode(self, proteins): | |
| return encode(proteins, self.pre, self.network) | |
| def reconstruct(self, proteins): | |
| z = self.encode(proteins) | |
| normalized = numeric_network(z, self.network, "decoder") | |
| return np.expm1(np.maximum(normalized*self.pre["scale"]+self.pre["mean"], 0.)) | |
| def predict(self, events): | |
| for event in events: | |
| if set(event) != {"visit_id", "visit_month", "proteins"}: | |
| raise ValueError("Only current visit_id, visit_month and proteins are accepted") | |
| if len(event["proteins"]) != len(self.spec["proteins"]): | |
| raise ValueError("Protein vocabulary mismatch") | |
| proteins = np.asarray([e["proteins"] for e in events], dtype=float) | |
| months = np.asarray([e["visit_month"] for e in events], dtype=float) | |
| z = self.encode(proteins) | |
| result = {} | |
| for arm, head in self.heads.items(): | |
| x = regression_features(months, proteins, z, arm, self.reg_pre) | |
| result[arm] = predict_heads(x, head["coef"], head["intercept"], head["multipliers"]) | |
| return result | |