Download code/ampd/latent_encoder.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 8.15 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/latent_encoder.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/latent_encoder.py
-
curl -L -o latent_encoder.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/latent_encoder.py
8.15 kB
| """Unsupervised autoregressive protein encoder with a shared 8D bottleneck.""" | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import warnings | |
| import numpy as np | |
| import torch | |
| from torch import nn | |
| CONFIG = {"model_dim": 32, "heads": 4, "layers": 1, "feedforward_dim": 64, | |
| "latent_dim": 8, "dropout": .1} | |
| def fit_scaler(proteins): | |
| log = np.log1p(np.asarray(proteins, dtype=float)) | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore", RuntimeWarning) | |
| mean, scale = np.nanmean(log, axis=0), np.nanstd(log, axis=0) | |
| mean = np.nan_to_num(mean) | |
| scale = np.where(np.isfinite(scale) & (scale > 1e-12), scale, 1.) | |
| return {"mean": mean, "scale": scale} | |
| def preprocess(proteins, params): | |
| proteins = np.asarray(proteins, dtype=float) | |
| if np.any(proteins < 0): | |
| raise ValueError("Protein abundance must be nonnegative") | |
| observed = np.isfinite(proteins) | |
| values = np.clip((np.log1p(proteins)-params["mean"])/params["scale"], -5., 5.) | |
| values = np.where(observed, values, 0.) | |
| return values.astype(np.float32), observed.astype(np.float32) | |
| class ProteinAR(nn.Module): | |
| def __init__(self, proteins, model_dim=32, heads=4, layers=1, feedforward_dim=64, latent_dim=8, dropout=.1): | |
| super().__init__() | |
| self.n_proteins = proteins | |
| self.identity = nn.Embedding(proteins+1, model_dim) | |
| self.value = nn.Linear(2, model_dim) | |
| layer = nn.TransformerEncoderLayer(model_dim, heads, feedforward_dim, dropout, | |
| activation="gelu", batch_first=True, norm_first=True) | |
| self.transformer = nn.TransformerEncoder(layer, layers, enable_nested_tensor=False) | |
| self.latent = nn.Sequential(nn.LayerNorm(model_dim), nn.Linear(model_dim, latent_dim)) | |
| self.target_identity = nn.Embedding(proteins, latent_dim) | |
| self.readout = nn.Sequential(nn.Linear(2*latent_dim, 32), nn.GELU(), nn.Linear(32, 1)) | |
| self.register_buffer("causal_mask", torch.triu(torch.ones(proteins+1, proteins+1, dtype=torch.bool), diagonal=1), persistent=False) | |
| def context(self, values, observed, order): | |
| if order.dim() == 1: | |
| order = order.unsqueeze(0).expand(len(values), -1) | |
| v, mask = values.gather(1, order), observed.gather(1, order) | |
| ids = torch.cat([torch.full((len(values), 1), self.n_proteins, dtype=torch.long, device=values.device), order], dim=1) | |
| measurements = torch.stack([v, mask], dim=-1) | |
| measurements = torch.cat([torch.zeros((len(values), 1, 2), dtype=values.dtype, device=values.device), measurements], dim=1) | |
| tokens = self.identity(ids)+self.value(measurements) | |
| z = self.latent(self.transformer(tokens, mask=self.causal_mask)) | |
| return z, order, v, mask | |
| def forward(self, values, observed, order): | |
| z, order, target, mask = self.context(values, observed, order) | |
| # Position k sees BOS plus proteins before k; the kth target value is masked out. | |
| prediction = self.readout(torch.cat([z[:, :-1], self.target_identity(order)], dim=-1)).squeeze(-1) | |
| return prediction, target, mask | |
| def encode(self, values, observed, order): | |
| return self.context(values, observed, order)[0][:, -1] | |
| def reconstruction_metrics(model, values, observed, order, batch=16): | |
| model.eval() | |
| totals = np.zeros(4, dtype=float) | |
| with torch.no_grad(): | |
| for start in range(0, len(values), batch): | |
| x = torch.tensor(values[start:start+batch]) | |
| m = torch.tensor(observed[start:start+batch]) | |
| prediction, target, mask = model(x, m, torch.tensor(order, dtype=torch.long)) | |
| huber = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.) | |
| totals += [float((huber*mask).sum()), float(((prediction-target)**2*mask).sum()), | |
| float((target**2*mask).sum()), float(mask.sum())] | |
| return {"huber": totals[0]/totals[3], "mse": totals[1]/totals[3], | |
| "training_mean_predictor_mse": totals[2]/totals[3], "observed_targets": int(totals[3])} | |
| def train_encoder(values, observed, config, seed, max_epochs, evaluation_order, | |
| validation=None, patience=15, schedule_epochs=120): | |
| torch.manual_seed(seed) | |
| model = ProteinAR(values.shape[1], **config) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=.001, weight_decay=.0001) | |
| scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=schedule_epochs, eta_min=.00001) | |
| x, mask = torch.tensor(values), torch.tensor(observed) | |
| rng = np.random.default_rng(seed) | |
| history, best, chosen, stale, best_state = [], float("inf"), max_epochs, 0, None | |
| for epoch in range(1, max_epochs+1): | |
| model.train() | |
| total, count = 0., 0. | |
| rows = rng.permutation(len(x)) | |
| for start in range(0, len(x), 16): | |
| index = rows[start:start+16] | |
| order = torch.tensor(np.argsort(rng.random((len(index), values.shape[1])), axis=1), dtype=torch.long) | |
| prediction, target, m = model(x[index], mask[index], order) | |
| per_value = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.) | |
| loss = (per_value*m).sum()/m.sum().clamp_min(1.) | |
| if not torch.isfinite(loss): | |
| raise ValueError("Nonfinite unsupervised loss") | |
| optimizer.zero_grad() | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.) | |
| optimizer.step() | |
| total += float((per_value.detach()*m).sum()) | |
| count += float(m.sum()) | |
| record = {"epoch": epoch, "train_huber": total/count} | |
| if validation is not None: | |
| metric = reconstruction_metrics(model, *validation, evaluation_order) | |
| record.update({f"validation_{k}": v for k, v in metric.items()}) | |
| if metric["huber"] < best-.0001: | |
| best, chosen, stale = metric["huber"], epoch, 0 | |
| best_state = {k: v.detach().clone() for k, v in model.state_dict().items()} | |
| else: | |
| stale += 1 | |
| history.append(record) | |
| scheduler.step() | |
| if epoch == 1 or epoch % 5 == 0: | |
| print(f"epoch={epoch}, train_huber={record['train_huber']:.4f}"+ | |
| (f", validation_huber={record['validation_huber']:.4f}" if validation is not None else ""), flush=True) | |
| if validation is not None and stale >= patience: | |
| break | |
| if best_state is not None: | |
| model.load_state_dict(best_state) | |
| return model.eval(), history, chosen | |
| def encode_values(model, values, observed, orders, batch=16): | |
| model.eval() | |
| output = [] | |
| with torch.no_grad(): | |
| for start in range(0, len(values), batch): | |
| x, m = torch.tensor(values[start:start+batch]), torch.tensor(observed[start:start+batch]) | |
| z = torch.stack([model.encode(x, m, torch.tensor(order, dtype=torch.long)) for order in orders]).mean(0) | |
| output.append(z.numpy()) | |
| return np.concatenate(output) | |
| class LatentPredictor: | |
| def __init__(self, bundle): | |
| bundle = Path(bundle) | |
| self.spec = json.loads((bundle/"model.json").read_text()) | |
| with np.load(bundle/"preprocessor.npz", allow_pickle=False) as archive: | |
| self.preprocessor = dict(archive) | |
| self.model = ProteinAR(len(self.spec["proteins"]), **self.spec["config"]) | |
| self.model.load_state_dict(torch.load(bundle/"weights.pt", map_location="cpu", weights_only=True)) | |
| self.model.eval() | |
| def predict(self, events): | |
| for event in events: | |
| if set(event) != {"visit_id", "visit_month", "proteins"}: | |
| raise ValueError("Only current visit metadata and proteins are accepted") | |
| if len(event["proteins"]) != len(self.spec["proteins"]): | |
| raise ValueError("Protein vocabulary mismatch") | |
| raw = np.asarray([e["proteins"] for e in events], dtype=float) | |
| values, mask = preprocess(raw, self.preprocessor) | |
| return encode_values(self.model, values, mask, self.spec["orders"]) | |