"""Unsupervised autoregressive protein encoder with a shared 8D bottleneck.""" from __future__ import annotations import json from pathlib import Path import warnings import numpy as np import torch from torch import nn CONFIG = {"model_dim": 32, "heads": 4, "layers": 1, "feedforward_dim": 64, "latent_dim": 8, "dropout": .1} def fit_scaler(proteins): log = np.log1p(np.asarray(proteins, dtype=float)) with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) mean, scale = np.nanmean(log, axis=0), np.nanstd(log, axis=0) mean = np.nan_to_num(mean) scale = np.where(np.isfinite(scale) & (scale > 1e-12), scale, 1.) return {"mean": mean, "scale": scale} def preprocess(proteins, params): proteins = np.asarray(proteins, dtype=float) if np.any(proteins < 0): raise ValueError("Protein abundance must be nonnegative") observed = np.isfinite(proteins) values = np.clip((np.log1p(proteins)-params["mean"])/params["scale"], -5., 5.) values = np.where(observed, values, 0.) return values.astype(np.float32), observed.astype(np.float32) class ProteinAR(nn.Module): def __init__(self, proteins, model_dim=32, heads=4, layers=1, feedforward_dim=64, latent_dim=8, dropout=.1): super().__init__() self.n_proteins = proteins self.identity = nn.Embedding(proteins+1, model_dim) self.value = nn.Linear(2, model_dim) layer = nn.TransformerEncoderLayer(model_dim, heads, feedforward_dim, dropout, activation="gelu", batch_first=True, norm_first=True) self.transformer = nn.TransformerEncoder(layer, layers, enable_nested_tensor=False) self.latent = nn.Sequential(nn.LayerNorm(model_dim), nn.Linear(model_dim, latent_dim)) self.target_identity = nn.Embedding(proteins, latent_dim) self.readout = nn.Sequential(nn.Linear(2*latent_dim, 32), nn.GELU(), nn.Linear(32, 1)) self.register_buffer("causal_mask", torch.triu(torch.ones(proteins+1, proteins+1, dtype=torch.bool), diagonal=1), persistent=False) def context(self, values, observed, order): if order.dim() == 1: order = order.unsqueeze(0).expand(len(values), -1) v, mask = values.gather(1, order), observed.gather(1, order) ids = torch.cat([torch.full((len(values), 1), self.n_proteins, dtype=torch.long, device=values.device), order], dim=1) measurements = torch.stack([v, mask], dim=-1) measurements = torch.cat([torch.zeros((len(values), 1, 2), dtype=values.dtype, device=values.device), measurements], dim=1) tokens = self.identity(ids)+self.value(measurements) z = self.latent(self.transformer(tokens, mask=self.causal_mask)) return z, order, v, mask def forward(self, values, observed, order): z, order, target, mask = self.context(values, observed, order) # Position k sees BOS plus proteins before k; the kth target value is masked out. prediction = self.readout(torch.cat([z[:, :-1], self.target_identity(order)], dim=-1)).squeeze(-1) return prediction, target, mask def encode(self, values, observed, order): return self.context(values, observed, order)[0][:, -1] def reconstruction_metrics(model, values, observed, order, batch=16): model.eval() totals = np.zeros(4, dtype=float) with torch.no_grad(): for start in range(0, len(values), batch): x = torch.tensor(values[start:start+batch]) m = torch.tensor(observed[start:start+batch]) prediction, target, mask = model(x, m, torch.tensor(order, dtype=torch.long)) huber = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.) totals += [float((huber*mask).sum()), float(((prediction-target)**2*mask).sum()), float((target**2*mask).sum()), float(mask.sum())] return {"huber": totals[0]/totals[3], "mse": totals[1]/totals[3], "training_mean_predictor_mse": totals[2]/totals[3], "observed_targets": int(totals[3])} def train_encoder(values, observed, config, seed, max_epochs, evaluation_order, validation=None, patience=15, schedule_epochs=120): torch.manual_seed(seed) model = ProteinAR(values.shape[1], **config) optimizer = torch.optim.AdamW(model.parameters(), lr=.001, weight_decay=.0001) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=schedule_epochs, eta_min=.00001) x, mask = torch.tensor(values), torch.tensor(observed) rng = np.random.default_rng(seed) history, best, chosen, stale, best_state = [], float("inf"), max_epochs, 0, None for epoch in range(1, max_epochs+1): model.train() total, count = 0., 0. rows = rng.permutation(len(x)) for start in range(0, len(x), 16): index = rows[start:start+16] order = torch.tensor(np.argsort(rng.random((len(index), values.shape[1])), axis=1), dtype=torch.long) prediction, target, m = model(x[index], mask[index], order) per_value = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.) loss = (per_value*m).sum()/m.sum().clamp_min(1.) if not torch.isfinite(loss): raise ValueError("Nonfinite unsupervised loss") optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.) optimizer.step() total += float((per_value.detach()*m).sum()) count += float(m.sum()) record = {"epoch": epoch, "train_huber": total/count} if validation is not None: metric = reconstruction_metrics(model, *validation, evaluation_order) record.update({f"validation_{k}": v for k, v in metric.items()}) if metric["huber"] < best-.0001: best, chosen, stale = metric["huber"], epoch, 0 best_state = {k: v.detach().clone() for k, v in model.state_dict().items()} else: stale += 1 history.append(record) scheduler.step() if epoch == 1 or epoch % 5 == 0: print(f"epoch={epoch}, train_huber={record['train_huber']:.4f}"+ (f", validation_huber={record['validation_huber']:.4f}" if validation is not None else ""), flush=True) if validation is not None and stale >= patience: break if best_state is not None: model.load_state_dict(best_state) return model.eval(), history, chosen def encode_values(model, values, observed, orders, batch=16): model.eval() output = [] with torch.no_grad(): for start in range(0, len(values), batch): x, m = torch.tensor(values[start:start+batch]), torch.tensor(observed[start:start+batch]) z = torch.stack([model.encode(x, m, torch.tensor(order, dtype=torch.long)) for order in orders]).mean(0) output.append(z.numpy()) return np.concatenate(output) class LatentPredictor: def __init__(self, bundle): bundle = Path(bundle) self.spec = json.loads((bundle/"model.json").read_text()) with np.load(bundle/"preprocessor.npz", allow_pickle=False) as archive: self.preprocessor = dict(archive) self.model = ProteinAR(len(self.spec["proteins"]), **self.spec["config"]) self.model.load_state_dict(torch.load(bundle/"weights.pt", map_location="cpu", weights_only=True)) self.model.eval() def predict(self, events): for event in events: if set(event) != {"visit_id", "visit_month", "proteins"}: raise ValueError("Only current visit metadata and proteins are accepted") if len(event["proteins"]) != len(self.spec["proteins"]): raise ValueError("Protein vocabulary mismatch") raw = np.asarray([e["proteins"] for e in events], dtype=float) values, mask = preprocess(raw, self.preprocessor) return encode_values(self.model, values, mask, self.spec["orders"])