"""Protein autoencoder, patient-level latent screening, and regression heads.""" from __future__ import annotations import json from pathlib import Path import warnings import numpy as np import pandas as pd import torch from scipy.stats import false_discovery_control, rankdata, t from sklearn.linear_model import Ridge LATENT = 32 MONTHS = np.array([0, 6, 12, 18, 24, 36, 48, 60, 72, 84]) ARMS = ("month", "missingness", "ae_all", "ae_topk", "ae_fdr", "ae_topk_month") ALPHAS = (1., 10., 100., 1000.) TOP_K = (2, 4, 8, 16) FACTORS = (0., .5, .75, 1.) class ProteinAutoencoder(torch.nn.Module): def __init__(self, n_proteins, latent=LATENT): super().__init__() self.encoder = torch.nn.Sequential(torch.nn.Linear(n_proteins, 128), torch.nn.ReLU(), torch.nn.Linear(128, 64), torch.nn.ReLU(), torch.nn.Linear(64, latent)) self.decoder = torch.nn.Sequential(torch.nn.Linear(latent, 64), torch.nn.ReLU(), torch.nn.Linear(64, 128), torch.nn.ReLU(), torch.nn.Linear(128, n_proteins)) def forward(self, x): return self.decoder(self.encoder(x)) def fit_preprocessor(proteins): values = np.asarray(proteins, dtype=float) log = np.where(np.isfinite(values), np.log1p(values), np.nan) with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) impute = np.nan_to_num(np.nanmedian(log, axis=0)) filled = np.where(np.isfinite(log), log, impute) mean, scale = filled.mean(0), filled.std(0) return {"impute": impute, "mean": mean, "scale": np.where(scale > 1e-12, scale, 1.)} def transform_proteins(proteins, params): values = np.asarray(proteins, dtype=float) if np.any(values < 0): raise ValueError("Protein abundance must be nonnegative") observed = np.isfinite(values) log = np.log1p(values) filled = np.where(observed, log, params["impute"]) return np.clip((filled-params["mean"])/params["scale"], -5., 5.), observed def masked_reconstruction_loss(prediction, target, observed): return (((prediction-target)**2)*observed).sum()/observed.sum().clamp_min(1.) def train_autoencoder(x, mask, seed, epochs, validation=None): torch.manual_seed(seed) model = ProteinAutoencoder(x.shape[1]) optimizer = torch.optim.AdamW(model.parameters(), lr=.001, weight_decay=.0001) xt, mt = torch.tensor(x, dtype=torch.float32), torch.tensor(mask, dtype=torch.float32) rng = np.random.default_rng(seed) best, chosen, stale, best_state = float("inf"), epochs, 0, None logs = [] if validation is not None: vx, vm = [torch.tensor(v, dtype=torch.float32) for v in validation] for epoch in range(1, epochs+1): model.train() total, count = 0., 0. permutation = rng.permutation(len(x)) for start in range(0, len(x), 64): rows = permutation[start:start+64] output = model(xt[rows]) loss = masked_reconstruction_loss(output, xt[rows], mt[rows]) if not torch.isfinite(loss): raise ValueError("Nonfinite reconstruction loss") optimizer.zero_grad() loss.backward() optimizer.step() n = float(mt[rows].sum()) total += float(loss.detach())*n count += n record = {"epoch": epoch, "train_observed_mse": total/count} if validation is not None: model.eval() with torch.no_grad(): value = float(masked_reconstruction_loss(model(vx), vx, vm)) record["validation_observed_mse"] = value if value < best-1e-5: best, chosen, stale = value, epoch, 0 best_state = {k: v.detach().clone() for k, v in model.state_dict().items()} else: stale += 1 logs.append(record) if validation is not None and stale >= 30: break if best_state is not None: model.load_state_dict(best_state) return model.eval(), pd.DataFrame(logs), chosen def export_numeric(model): return {key.replace(".", "_"): value.detach().numpy().astype(float).copy() for key, value in model.state_dict().items()} def numeric_network(x, params, prefix): result = np.asarray(x, dtype=float) for layer in [0, 2, 4]: result = result @ params[f"{prefix}_{layer}_weight"].T + params[f"{prefix}_{layer}_bias"] if layer != 4: result = np.maximum(result, 0) return result def encode(proteins, preprocessor, network): x, _ = transform_proteins(proteins, preprocessor) return numeric_network(x, network, "encoder") def reconstruction_metrics(proteins, preprocessor, network): x, mask = transform_proteins(proteins, preprocessor) reconstructed = numeric_network(numeric_network(x, network, "encoder"), network, "decoder") error = (reconstructed-x)[mask] return {"n_samples": len(x), "n_observed_values": int(mask.sum()), "observed_mse": float(np.mean(error**2)), "observed_mae": float(np.mean(np.abs(error))), "training_mean_baseline_mse": float(np.mean(x[mask]**2))} def errors(y, prediction): return 200*np.abs(y-prediction)/(np.abs(y)+np.abs(prediction)+2) def centered_ranks(x): r = rankdata(x, axis=0, method="average") r -= r.mean(0) norm = np.sqrt((r*r).sum(0)) return np.divide(r, norm, out=np.zeros_like(r), where=norm > 0) def partial_rank(x, y, month, detected): rx, ry = rankdata(x), rankdata(y) tm, dc = rankdata(month), rankdata(detected) tm = (tm-tm.mean())/max(tm.std(), 1.) dc = (dc-dc.mean())/max(dc.std(), 1.) z = np.c_[np.ones(len(y)), tm, tm**2, dc] u, s, _ = np.linalg.svd(z, full_matrices=False) q = u[:, s > s[0]*max(z.shape)*np.finfo(float).eps] a, b = rx-q@(q.T@rx), ry-q@(q.T@ry) xx, yy = a@a, b@b if xx < 1e-12 or yy < 1e-12: return np.nan, np.nan coefficient = (a@b)/xx residual = b-coefficient*a leverage = (q*q).sum(1)+a*a/xx if np.max(leverage) >= 1-1e-10: return float((a@b)/np.sqrt(xx*yy)), np.nan variance = np.sum((a*residual/(1-leverage))**2)/xx**2 p = 2*t.sf(abs(coefficient)/np.sqrt(max(variance, 1e-30)), len(y)-q.shape[1]-1) return float((a@b)/np.sqrt(xx*yy)), float(p) def screen_latents(z, samples, y, permutations=99999, seed=42): """One earliest eligible visit per patient/target; never pools repeated rows.""" records = [] for target in range(4): meta = samples.copy().reset_index(drop=True) meta["row"] = np.arange(len(meta)) meta["outcome"] = y[:, target*4] first = meta.loc[meta.outcome.notna()].sort_values(["uid", "visit_month"]).drop_duplicates("uid") index = first.row.to_numpy(int) values = first.outcome.to_numpy(float) a, b = centered_ranks(z[index]), centered_ranks(values) rho = a.T @ b count = np.zeros(z.shape[1], dtype=np.int64) rng = np.random.default_rng(seed+target) for start in range(0, permutations, 2048): size = min(2048, permutations-start) shuffled = rng.permuted(np.broadcast_to(b, (size, len(b))), axis=1) count += (np.abs(shuffled@a) >= np.abs(rho)[None, :]-1e-12).sum(0) p = (count+1)/(permutations+1) for coordinate in range(z.shape[1]): active = np.std(z[index, coordinate]) > 1e-10 and np.unique(values).size >= 3 and len(index) >= 30 r_adjusted, p_adjusted = partial_rank(z[index, coordinate], values, first.visit_month.to_numpy(), first.detected_count.to_numpy()) if active else (np.nan, np.nan) records.append({"target": f"updrs_{target+1}", "latent": coordinate, "n_patients": len(index), "rho": float(rho[coordinate]) if active else np.nan, "p_permutation": float(p[coordinate]) if active else np.nan, "rho_adjusted": r_adjusted, "p_adjusted": p_adjusted, "rho_detected_count": float(centered_ranks(z[index, coordinate]) @ centered_ranks(first.detected_count.to_numpy())), "rho_month": float(centered_ranks(z[index, coordinate]) @ centered_ranks(first.visit_month.to_numpy()))}) result = pd.DataFrame(records) for pcol, qcol in [("p_permutation", "q_bh"), ("p_adjusted", "q_adjusted_bh")]: valid = np.isfinite(result[pcol]) result[qcol] = np.where(valid, false_discovery_control(np.where(valid, result[pcol], 1.), method="bh"), np.nan) return result def selection_mask(correlations, policy, k=0): mask = np.zeros((4, LATENT), dtype=bool) for target in range(4): rows = correlations.loc[correlations.target == f"updrs_{target+1}"].copy() rows["absolute"] = rows.rho.abs() if policy == "all": selected = np.arange(LATENT) elif policy == "fdr": selected = rows.loc[(rows.q_bh < .05) & (rows.absolute >= .2), "latent"].to_numpy(int) elif policy == "topk": selected = rows.loc[rows.rho.notna()].sort_values(["absolute", "latent"], ascending=[False, True]).head(k).latent.to_numpy(int) else: selected = np.empty(0, dtype=int) mask[target, selected] = True return mask def month_features(month): month = np.asarray(month, dtype=float) return np.c_[month/84., (month/84.)**2, month[:, None] == MONTHS] def quality_features(proteins): missing = ~np.isfinite(proteins) return np.c_[missing, (~missing).sum(1)].astype(float) def fit_regression_pre(months, proteins, latent): result = {} for name, values in [("month", month_features(months)), ("quality", quality_features(proteins)), ("latent", latent)]: mean, std = values.mean(0), values.std(0) result[f"{name}_mean"], result[f"{name}_scale"] = mean, np.where(std > 1e-12, std, 1.) return result def regression_features(months, proteins, latent, arm, pre): if arm == "month": return (month_features(months)-pre["month_mean"])/pre["month_scale"] if arm == "missingness": return (quality_features(proteins)-pre["quality_mean"])/pre["quality_scale"] z = (latent-pre["latent_mean"])/pre["latent_scale"] if arm == "ae_topk_month": m = (month_features(months)-pre["month_mean"])/pre["month_scale"] return np.c_[m, z] return z def fit_heads(x, y, arm, mask, alpha): coefficients, intercepts = [], [] for head in range(16): active = np.ones(x.shape[1], dtype=bool) if arm.startswith("ae_"): active[-LATENT:] = mask[head//4] valid = np.isfinite(y[:, head]) model = Ridge(alpha=alpha, solver="cholesky").fit(x[valid]*active, y[valid, head]) coefficients.append(model.coef_*active) intercepts.append(model.intercept_) return np.stack(coefficients), np.asarray(intercepts) def predict_heads(x, coef, intercept, multipliers): return np.clip(x@coef.T+intercept, 0., 86.) * np.repeat(multipliers, 4) def calibrate(y, prediction): multipliers = [] for target in range(4): cols = slice(target*4, (target+1)*4) scores = [float(np.nanmean(errors(y[:, cols], prediction[:, cols]*factor))) for factor in FACTORS] multipliers.append(FACTORS[int(np.argmin(scores))]) return multipliers, float(np.nanmean(errors(y, prediction*np.repeat(multipliers, 4)))) class AELatentPredictor: def __init__(self, bundle): bundle = Path(bundle) self.spec = json.loads((bundle/"model.json").read_text()) self.pre = dict(np.load(bundle/"protein_preprocessor.npz", allow_pickle=False)) self.network = dict(np.load(bundle/"autoencoder_numeric.npz", allow_pickle=False)) self.reg_pre = dict(np.load(bundle/"regression_preprocessor.npz", allow_pickle=False)) self.heads = {arm: dict(np.load(bundle/f"{arm}.npz", allow_pickle=False)) for arm in ARMS} def encode(self, proteins): return encode(proteins, self.pre, self.network) def reconstruct(self, proteins): z = self.encode(proteins) normalized = numeric_network(z, self.network, "decoder") return np.expm1(np.maximum(normalized*self.pre["scale"]+self.pre["mean"], 0.)) def predict(self, events): for event in events: if set(event) != {"visit_id", "visit_month", "proteins"}: raise ValueError("Only current visit_id, visit_month and proteins are accepted") if len(event["proteins"]) != len(self.spec["proteins"]): raise ValueError("Protein vocabulary mismatch") proteins = np.asarray([e["proteins"] for e in events], dtype=float) months = np.asarray([e["visit_month"] for e in events], dtype=float) z = self.encode(proteins) result = {} for arm, head in self.heads.items(): x = regression_features(months, proteins, z, arm, self.reg_pre) result[arm] = predict_heads(x, head["coef"], head["intercept"], head["multipliers"]) return result