cs4486-amp-pd-checkpoints / code /ampd /ae_latent_selection.py
fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
13 kB
"""Protein autoencoder, patient-level latent screening, and regression heads."""
from __future__ import annotations
import json
from pathlib import Path
import warnings
import numpy as np
import pandas as pd
import torch
from scipy.stats import false_discovery_control, rankdata, t
from sklearn.linear_model import Ridge
LATENT = 32
MONTHS = np.array([0, 6, 12, 18, 24, 36, 48, 60, 72, 84])
ARMS = ("month", "missingness", "ae_all", "ae_topk", "ae_fdr", "ae_topk_month")
ALPHAS = (1., 10., 100., 1000.)
TOP_K = (2, 4, 8, 16)
FACTORS = (0., .5, .75, 1.)
class ProteinAutoencoder(torch.nn.Module):
def __init__(self, n_proteins, latent=LATENT):
super().__init__()
self.encoder = torch.nn.Sequential(torch.nn.Linear(n_proteins, 128), torch.nn.ReLU(),
torch.nn.Linear(128, 64), torch.nn.ReLU(), torch.nn.Linear(64, latent))
self.decoder = torch.nn.Sequential(torch.nn.Linear(latent, 64), torch.nn.ReLU(),
torch.nn.Linear(64, 128), torch.nn.ReLU(), torch.nn.Linear(128, n_proteins))
def forward(self, x):
return self.decoder(self.encoder(x))
def fit_preprocessor(proteins):
values = np.asarray(proteins, dtype=float)
log = np.where(np.isfinite(values), np.log1p(values), np.nan)
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
impute = np.nan_to_num(np.nanmedian(log, axis=0))
filled = np.where(np.isfinite(log), log, impute)
mean, scale = filled.mean(0), filled.std(0)
return {"impute": impute, "mean": mean, "scale": np.where(scale > 1e-12, scale, 1.)}
def transform_proteins(proteins, params):
values = np.asarray(proteins, dtype=float)
if np.any(values < 0):
raise ValueError("Protein abundance must be nonnegative")
observed = np.isfinite(values)
log = np.log1p(values)
filled = np.where(observed, log, params["impute"])
return np.clip((filled-params["mean"])/params["scale"], -5., 5.), observed
def masked_reconstruction_loss(prediction, target, observed):
return (((prediction-target)**2)*observed).sum()/observed.sum().clamp_min(1.)
def train_autoencoder(x, mask, seed, epochs, validation=None):
torch.manual_seed(seed)
model = ProteinAutoencoder(x.shape[1])
optimizer = torch.optim.AdamW(model.parameters(), lr=.001, weight_decay=.0001)
xt, mt = torch.tensor(x, dtype=torch.float32), torch.tensor(mask, dtype=torch.float32)
rng = np.random.default_rng(seed)
best, chosen, stale, best_state = float("inf"), epochs, 0, None
logs = []
if validation is not None:
vx, vm = [torch.tensor(v, dtype=torch.float32) for v in validation]
for epoch in range(1, epochs+1):
model.train()
total, count = 0., 0.
permutation = rng.permutation(len(x))
for start in range(0, len(x), 64):
rows = permutation[start:start+64]
output = model(xt[rows])
loss = masked_reconstruction_loss(output, xt[rows], mt[rows])
if not torch.isfinite(loss):
raise ValueError("Nonfinite reconstruction loss")
optimizer.zero_grad()
loss.backward()
optimizer.step()
n = float(mt[rows].sum())
total += float(loss.detach())*n
count += n
record = {"epoch": epoch, "train_observed_mse": total/count}
if validation is not None:
model.eval()
with torch.no_grad():
value = float(masked_reconstruction_loss(model(vx), vx, vm))
record["validation_observed_mse"] = value
if value < best-1e-5:
best, chosen, stale = value, epoch, 0
best_state = {k: v.detach().clone() for k, v in model.state_dict().items()}
else:
stale += 1
logs.append(record)
if validation is not None and stale >= 30:
break
if best_state is not None:
model.load_state_dict(best_state)
return model.eval(), pd.DataFrame(logs), chosen
def export_numeric(model):
return {key.replace(".", "_"): value.detach().numpy().astype(float).copy()
for key, value in model.state_dict().items()}
def numeric_network(x, params, prefix):
result = np.asarray(x, dtype=float)
for layer in [0, 2, 4]:
result = result @ params[f"{prefix}_{layer}_weight"].T + params[f"{prefix}_{layer}_bias"]
if layer != 4:
result = np.maximum(result, 0)
return result
def encode(proteins, preprocessor, network):
x, _ = transform_proteins(proteins, preprocessor)
return numeric_network(x, network, "encoder")
def reconstruction_metrics(proteins, preprocessor, network):
x, mask = transform_proteins(proteins, preprocessor)
reconstructed = numeric_network(numeric_network(x, network, "encoder"), network, "decoder")
error = (reconstructed-x)[mask]
return {"n_samples": len(x), "n_observed_values": int(mask.sum()),
"observed_mse": float(np.mean(error**2)), "observed_mae": float(np.mean(np.abs(error))),
"training_mean_baseline_mse": float(np.mean(x[mask]**2))}
def errors(y, prediction):
return 200*np.abs(y-prediction)/(np.abs(y)+np.abs(prediction)+2)
def centered_ranks(x):
r = rankdata(x, axis=0, method="average")
r -= r.mean(0)
norm = np.sqrt((r*r).sum(0))
return np.divide(r, norm, out=np.zeros_like(r), where=norm > 0)
def partial_rank(x, y, month, detected):
rx, ry = rankdata(x), rankdata(y)
tm, dc = rankdata(month), rankdata(detected)
tm = (tm-tm.mean())/max(tm.std(), 1.)
dc = (dc-dc.mean())/max(dc.std(), 1.)
z = np.c_[np.ones(len(y)), tm, tm**2, dc]
u, s, _ = np.linalg.svd(z, full_matrices=False)
q = u[:, s > s[0]*max(z.shape)*np.finfo(float).eps]
a, b = rx-q@(q.T@rx), ry-q@(q.T@ry)
xx, yy = a@a, b@b
if xx < 1e-12 or yy < 1e-12:
return np.nan, np.nan
coefficient = (a@b)/xx
residual = b-coefficient*a
leverage = (q*q).sum(1)+a*a/xx
if np.max(leverage) >= 1-1e-10:
return float((a@b)/np.sqrt(xx*yy)), np.nan
variance = np.sum((a*residual/(1-leverage))**2)/xx**2
p = 2*t.sf(abs(coefficient)/np.sqrt(max(variance, 1e-30)), len(y)-q.shape[1]-1)
return float((a@b)/np.sqrt(xx*yy)), float(p)
def screen_latents(z, samples, y, permutations=99999, seed=42):
"""One earliest eligible visit per patient/target; never pools repeated rows."""
records = []
for target in range(4):
meta = samples.copy().reset_index(drop=True)
meta["row"] = np.arange(len(meta))
meta["outcome"] = y[:, target*4]
first = meta.loc[meta.outcome.notna()].sort_values(["uid", "visit_month"]).drop_duplicates("uid")
index = first.row.to_numpy(int)
values = first.outcome.to_numpy(float)
a, b = centered_ranks(z[index]), centered_ranks(values)
rho = a.T @ b
count = np.zeros(z.shape[1], dtype=np.int64)
rng = np.random.default_rng(seed+target)
for start in range(0, permutations, 2048):
size = min(2048, permutations-start)
shuffled = rng.permuted(np.broadcast_to(b, (size, len(b))), axis=1)
count += (np.abs(shuffled@a) >= np.abs(rho)[None, :]-1e-12).sum(0)
p = (count+1)/(permutations+1)
for coordinate in range(z.shape[1]):
active = np.std(z[index, coordinate]) > 1e-10 and np.unique(values).size >= 3 and len(index) >= 30
r_adjusted, p_adjusted = partial_rank(z[index, coordinate], values,
first.visit_month.to_numpy(), first.detected_count.to_numpy()) if active else (np.nan, np.nan)
records.append({"target": f"updrs_{target+1}", "latent": coordinate, "n_patients": len(index),
"rho": float(rho[coordinate]) if active else np.nan, "p_permutation": float(p[coordinate]) if active else np.nan,
"rho_adjusted": r_adjusted, "p_adjusted": p_adjusted,
"rho_detected_count": float(centered_ranks(z[index, coordinate]) @ centered_ranks(first.detected_count.to_numpy())),
"rho_month": float(centered_ranks(z[index, coordinate]) @ centered_ranks(first.visit_month.to_numpy()))})
result = pd.DataFrame(records)
for pcol, qcol in [("p_permutation", "q_bh"), ("p_adjusted", "q_adjusted_bh")]:
valid = np.isfinite(result[pcol])
result[qcol] = np.where(valid, false_discovery_control(np.where(valid, result[pcol], 1.), method="bh"), np.nan)
return result
def selection_mask(correlations, policy, k=0):
mask = np.zeros((4, LATENT), dtype=bool)
for target in range(4):
rows = correlations.loc[correlations.target == f"updrs_{target+1}"].copy()
rows["absolute"] = rows.rho.abs()
if policy == "all":
selected = np.arange(LATENT)
elif policy == "fdr":
selected = rows.loc[(rows.q_bh < .05) & (rows.absolute >= .2), "latent"].to_numpy(int)
elif policy == "topk":
selected = rows.loc[rows.rho.notna()].sort_values(["absolute", "latent"], ascending=[False, True]).head(k).latent.to_numpy(int)
else:
selected = np.empty(0, dtype=int)
mask[target, selected] = True
return mask
def month_features(month):
month = np.asarray(month, dtype=float)
return np.c_[month/84., (month/84.)**2, month[:, None] == MONTHS]
def quality_features(proteins):
missing = ~np.isfinite(proteins)
return np.c_[missing, (~missing).sum(1)].astype(float)
def fit_regression_pre(months, proteins, latent):
result = {}
for name, values in [("month", month_features(months)), ("quality", quality_features(proteins)), ("latent", latent)]:
mean, std = values.mean(0), values.std(0)
result[f"{name}_mean"], result[f"{name}_scale"] = mean, np.where(std > 1e-12, std, 1.)
return result
def regression_features(months, proteins, latent, arm, pre):
if arm == "month":
return (month_features(months)-pre["month_mean"])/pre["month_scale"]
if arm == "missingness":
return (quality_features(proteins)-pre["quality_mean"])/pre["quality_scale"]
z = (latent-pre["latent_mean"])/pre["latent_scale"]
if arm == "ae_topk_month":
m = (month_features(months)-pre["month_mean"])/pre["month_scale"]
return np.c_[m, z]
return z
def fit_heads(x, y, arm, mask, alpha):
coefficients, intercepts = [], []
for head in range(16):
active = np.ones(x.shape[1], dtype=bool)
if arm.startswith("ae_"):
active[-LATENT:] = mask[head//4]
valid = np.isfinite(y[:, head])
model = Ridge(alpha=alpha, solver="cholesky").fit(x[valid]*active, y[valid, head])
coefficients.append(model.coef_*active)
intercepts.append(model.intercept_)
return np.stack(coefficients), np.asarray(intercepts)
def predict_heads(x, coef, intercept, multipliers):
return np.clip(x@coef.T+intercept, 0., 86.) * np.repeat(multipliers, 4)
def calibrate(y, prediction):
multipliers = []
for target in range(4):
cols = slice(target*4, (target+1)*4)
scores = [float(np.nanmean(errors(y[:, cols], prediction[:, cols]*factor))) for factor in FACTORS]
multipliers.append(FACTORS[int(np.argmin(scores))])
return multipliers, float(np.nanmean(errors(y, prediction*np.repeat(multipliers, 4))))
class AELatentPredictor:
def __init__(self, bundle):
bundle = Path(bundle)
self.spec = json.loads((bundle/"model.json").read_text())
self.pre = dict(np.load(bundle/"protein_preprocessor.npz", allow_pickle=False))
self.network = dict(np.load(bundle/"autoencoder_numeric.npz", allow_pickle=False))
self.reg_pre = dict(np.load(bundle/"regression_preprocessor.npz", allow_pickle=False))
self.heads = {arm: dict(np.load(bundle/f"{arm}.npz", allow_pickle=False)) for arm in ARMS}
def encode(self, proteins):
return encode(proteins, self.pre, self.network)
def reconstruct(self, proteins):
z = self.encode(proteins)
normalized = numeric_network(z, self.network, "decoder")
return np.expm1(np.maximum(normalized*self.pre["scale"]+self.pre["mean"], 0.))
def predict(self, events):
for event in events:
if set(event) != {"visit_id", "visit_month", "proteins"}:
raise ValueError("Only current visit_id, visit_month and proteins are accepted")
if len(event["proteins"]) != len(self.spec["proteins"]):
raise ValueError("Protein vocabulary mismatch")
proteins = np.asarray([e["proteins"] for e in events], dtype=float)
months = np.asarray([e["visit_month"] for e in events], dtype=float)
z = self.encode(proteins)
result = {}
for arm, head in self.heads.items():
x = regression_features(months, proteins, z, arm, self.reg_pre)
result[arm] = predict_heads(x, head["coef"], head["intercept"], head["multipliers"])
return result