cs4486-amp-pd-checkpoints / code /ampd /latent_encoder.py
fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
8.15 kB
"""Unsupervised autoregressive protein encoder with a shared 8D bottleneck."""
from __future__ import annotations
import json
from pathlib import Path
import warnings
import numpy as np
import torch
from torch import nn
CONFIG = {"model_dim": 32, "heads": 4, "layers": 1, "feedforward_dim": 64,
"latent_dim": 8, "dropout": .1}
def fit_scaler(proteins):
log = np.log1p(np.asarray(proteins, dtype=float))
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
mean, scale = np.nanmean(log, axis=0), np.nanstd(log, axis=0)
mean = np.nan_to_num(mean)
scale = np.where(np.isfinite(scale) & (scale > 1e-12), scale, 1.)
return {"mean": mean, "scale": scale}
def preprocess(proteins, params):
proteins = np.asarray(proteins, dtype=float)
if np.any(proteins < 0):
raise ValueError("Protein abundance must be nonnegative")
observed = np.isfinite(proteins)
values = np.clip((np.log1p(proteins)-params["mean"])/params["scale"], -5., 5.)
values = np.where(observed, values, 0.)
return values.astype(np.float32), observed.astype(np.float32)
class ProteinAR(nn.Module):
def __init__(self, proteins, model_dim=32, heads=4, layers=1, feedforward_dim=64, latent_dim=8, dropout=.1):
super().__init__()
self.n_proteins = proteins
self.identity = nn.Embedding(proteins+1, model_dim)
self.value = nn.Linear(2, model_dim)
layer = nn.TransformerEncoderLayer(model_dim, heads, feedforward_dim, dropout,
activation="gelu", batch_first=True, norm_first=True)
self.transformer = nn.TransformerEncoder(layer, layers, enable_nested_tensor=False)
self.latent = nn.Sequential(nn.LayerNorm(model_dim), nn.Linear(model_dim, latent_dim))
self.target_identity = nn.Embedding(proteins, latent_dim)
self.readout = nn.Sequential(nn.Linear(2*latent_dim, 32), nn.GELU(), nn.Linear(32, 1))
self.register_buffer("causal_mask", torch.triu(torch.ones(proteins+1, proteins+1, dtype=torch.bool), diagonal=1), persistent=False)
def context(self, values, observed, order):
if order.dim() == 1:
order = order.unsqueeze(0).expand(len(values), -1)
v, mask = values.gather(1, order), observed.gather(1, order)
ids = torch.cat([torch.full((len(values), 1), self.n_proteins, dtype=torch.long, device=values.device), order], dim=1)
measurements = torch.stack([v, mask], dim=-1)
measurements = torch.cat([torch.zeros((len(values), 1, 2), dtype=values.dtype, device=values.device), measurements], dim=1)
tokens = self.identity(ids)+self.value(measurements)
z = self.latent(self.transformer(tokens, mask=self.causal_mask))
return z, order, v, mask
def forward(self, values, observed, order):
z, order, target, mask = self.context(values, observed, order)
# Position k sees BOS plus proteins before k; the kth target value is masked out.
prediction = self.readout(torch.cat([z[:, :-1], self.target_identity(order)], dim=-1)).squeeze(-1)
return prediction, target, mask
def encode(self, values, observed, order):
return self.context(values, observed, order)[0][:, -1]
def reconstruction_metrics(model, values, observed, order, batch=16):
model.eval()
totals = np.zeros(4, dtype=float)
with torch.no_grad():
for start in range(0, len(values), batch):
x = torch.tensor(values[start:start+batch])
m = torch.tensor(observed[start:start+batch])
prediction, target, mask = model(x, m, torch.tensor(order, dtype=torch.long))
huber = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.)
totals += [float((huber*mask).sum()), float(((prediction-target)**2*mask).sum()),
float((target**2*mask).sum()), float(mask.sum())]
return {"huber": totals[0]/totals[3], "mse": totals[1]/totals[3],
"training_mean_predictor_mse": totals[2]/totals[3], "observed_targets": int(totals[3])}
def train_encoder(values, observed, config, seed, max_epochs, evaluation_order,
validation=None, patience=15, schedule_epochs=120):
torch.manual_seed(seed)
model = ProteinAR(values.shape[1], **config)
optimizer = torch.optim.AdamW(model.parameters(), lr=.001, weight_decay=.0001)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=schedule_epochs, eta_min=.00001)
x, mask = torch.tensor(values), torch.tensor(observed)
rng = np.random.default_rng(seed)
history, best, chosen, stale, best_state = [], float("inf"), max_epochs, 0, None
for epoch in range(1, max_epochs+1):
model.train()
total, count = 0., 0.
rows = rng.permutation(len(x))
for start in range(0, len(x), 16):
index = rows[start:start+16]
order = torch.tensor(np.argsort(rng.random((len(index), values.shape[1])), axis=1), dtype=torch.long)
prediction, target, m = model(x[index], mask[index], order)
per_value = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.)
loss = (per_value*m).sum()/m.sum().clamp_min(1.)
if not torch.isfinite(loss):
raise ValueError("Nonfinite unsupervised loss")
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
optimizer.step()
total += float((per_value.detach()*m).sum())
count += float(m.sum())
record = {"epoch": epoch, "train_huber": total/count}
if validation is not None:
metric = reconstruction_metrics(model, *validation, evaluation_order)
record.update({f"validation_{k}": v for k, v in metric.items()})
if metric["huber"] < best-.0001:
best, chosen, stale = metric["huber"], epoch, 0
best_state = {k: v.detach().clone() for k, v in model.state_dict().items()}
else:
stale += 1
history.append(record)
scheduler.step()
if epoch == 1 or epoch % 5 == 0:
print(f"epoch={epoch}, train_huber={record['train_huber']:.4f}"+
(f", validation_huber={record['validation_huber']:.4f}" if validation is not None else ""), flush=True)
if validation is not None and stale >= patience:
break
if best_state is not None:
model.load_state_dict(best_state)
return model.eval(), history, chosen
def encode_values(model, values, observed, orders, batch=16):
model.eval()
output = []
with torch.no_grad():
for start in range(0, len(values), batch):
x, m = torch.tensor(values[start:start+batch]), torch.tensor(observed[start:start+batch])
z = torch.stack([model.encode(x, m, torch.tensor(order, dtype=torch.long)) for order in orders]).mean(0)
output.append(z.numpy())
return np.concatenate(output)
class LatentPredictor:
def __init__(self, bundle):
bundle = Path(bundle)
self.spec = json.loads((bundle/"model.json").read_text())
with np.load(bundle/"preprocessor.npz", allow_pickle=False) as archive:
self.preprocessor = dict(archive)
self.model = ProteinAR(len(self.spec["proteins"]), **self.spec["config"])
self.model.load_state_dict(torch.load(bundle/"weights.pt", map_location="cpu", weights_only=True))
self.model.eval()
def predict(self, events):
for event in events:
if set(event) != {"visit_id", "visit_month", "proteins"}:
raise ValueError("Only current visit metadata and proteins are accepted")
if len(event["proteins"]) != len(self.spec["proteins"]):
raise ValueError("Protein vocabulary mismatch")
raw = np.asarray([e["proteins"] for e in events], dtype=float)
values, mask = preprocess(raw, self.preprocessor)
return encode_values(self.model, values, mask, self.spec["orders"])