File size: 8,151 Bytes
04ae9bb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | """Unsupervised autoregressive protein encoder with a shared 8D bottleneck."""
from __future__ import annotations
import json
from pathlib import Path
import warnings
import numpy as np
import torch
from torch import nn
CONFIG = {"model_dim": 32, "heads": 4, "layers": 1, "feedforward_dim": 64,
"latent_dim": 8, "dropout": .1}
def fit_scaler(proteins):
log = np.log1p(np.asarray(proteins, dtype=float))
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
mean, scale = np.nanmean(log, axis=0), np.nanstd(log, axis=0)
mean = np.nan_to_num(mean)
scale = np.where(np.isfinite(scale) & (scale > 1e-12), scale, 1.)
return {"mean": mean, "scale": scale}
def preprocess(proteins, params):
proteins = np.asarray(proteins, dtype=float)
if np.any(proteins < 0):
raise ValueError("Protein abundance must be nonnegative")
observed = np.isfinite(proteins)
values = np.clip((np.log1p(proteins)-params["mean"])/params["scale"], -5., 5.)
values = np.where(observed, values, 0.)
return values.astype(np.float32), observed.astype(np.float32)
class ProteinAR(nn.Module):
def __init__(self, proteins, model_dim=32, heads=4, layers=1, feedforward_dim=64, latent_dim=8, dropout=.1):
super().__init__()
self.n_proteins = proteins
self.identity = nn.Embedding(proteins+1, model_dim)
self.value = nn.Linear(2, model_dim)
layer = nn.TransformerEncoderLayer(model_dim, heads, feedforward_dim, dropout,
activation="gelu", batch_first=True, norm_first=True)
self.transformer = nn.TransformerEncoder(layer, layers, enable_nested_tensor=False)
self.latent = nn.Sequential(nn.LayerNorm(model_dim), nn.Linear(model_dim, latent_dim))
self.target_identity = nn.Embedding(proteins, latent_dim)
self.readout = nn.Sequential(nn.Linear(2*latent_dim, 32), nn.GELU(), nn.Linear(32, 1))
self.register_buffer("causal_mask", torch.triu(torch.ones(proteins+1, proteins+1, dtype=torch.bool), diagonal=1), persistent=False)
def context(self, values, observed, order):
if order.dim() == 1:
order = order.unsqueeze(0).expand(len(values), -1)
v, mask = values.gather(1, order), observed.gather(1, order)
ids = torch.cat([torch.full((len(values), 1), self.n_proteins, dtype=torch.long, device=values.device), order], dim=1)
measurements = torch.stack([v, mask], dim=-1)
measurements = torch.cat([torch.zeros((len(values), 1, 2), dtype=values.dtype, device=values.device), measurements], dim=1)
tokens = self.identity(ids)+self.value(measurements)
z = self.latent(self.transformer(tokens, mask=self.causal_mask))
return z, order, v, mask
def forward(self, values, observed, order):
z, order, target, mask = self.context(values, observed, order)
# Position k sees BOS plus proteins before k; the kth target value is masked out.
prediction = self.readout(torch.cat([z[:, :-1], self.target_identity(order)], dim=-1)).squeeze(-1)
return prediction, target, mask
def encode(self, values, observed, order):
return self.context(values, observed, order)[0][:, -1]
def reconstruction_metrics(model, values, observed, order, batch=16):
model.eval()
totals = np.zeros(4, dtype=float)
with torch.no_grad():
for start in range(0, len(values), batch):
x = torch.tensor(values[start:start+batch])
m = torch.tensor(observed[start:start+batch])
prediction, target, mask = model(x, m, torch.tensor(order, dtype=torch.long))
huber = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.)
totals += [float((huber*mask).sum()), float(((prediction-target)**2*mask).sum()),
float((target**2*mask).sum()), float(mask.sum())]
return {"huber": totals[0]/totals[3], "mse": totals[1]/totals[3],
"training_mean_predictor_mse": totals[2]/totals[3], "observed_targets": int(totals[3])}
def train_encoder(values, observed, config, seed, max_epochs, evaluation_order,
validation=None, patience=15, schedule_epochs=120):
torch.manual_seed(seed)
model = ProteinAR(values.shape[1], **config)
optimizer = torch.optim.AdamW(model.parameters(), lr=.001, weight_decay=.0001)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=schedule_epochs, eta_min=.00001)
x, mask = torch.tensor(values), torch.tensor(observed)
rng = np.random.default_rng(seed)
history, best, chosen, stale, best_state = [], float("inf"), max_epochs, 0, None
for epoch in range(1, max_epochs+1):
model.train()
total, count = 0., 0.
rows = rng.permutation(len(x))
for start in range(0, len(x), 16):
index = rows[start:start+16]
order = torch.tensor(np.argsort(rng.random((len(index), values.shape[1])), axis=1), dtype=torch.long)
prediction, target, m = model(x[index], mask[index], order)
per_value = torch.nn.functional.huber_loss(prediction, target, reduction="none", delta=1.)
loss = (per_value*m).sum()/m.sum().clamp_min(1.)
if not torch.isfinite(loss):
raise ValueError("Nonfinite unsupervised loss")
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
optimizer.step()
total += float((per_value.detach()*m).sum())
count += float(m.sum())
record = {"epoch": epoch, "train_huber": total/count}
if validation is not None:
metric = reconstruction_metrics(model, *validation, evaluation_order)
record.update({f"validation_{k}": v for k, v in metric.items()})
if metric["huber"] < best-.0001:
best, chosen, stale = metric["huber"], epoch, 0
best_state = {k: v.detach().clone() for k, v in model.state_dict().items()}
else:
stale += 1
history.append(record)
scheduler.step()
if epoch == 1 or epoch % 5 == 0:
print(f"epoch={epoch}, train_huber={record['train_huber']:.4f}"+
(f", validation_huber={record['validation_huber']:.4f}" if validation is not None else ""), flush=True)
if validation is not None and stale >= patience:
break
if best_state is not None:
model.load_state_dict(best_state)
return model.eval(), history, chosen
def encode_values(model, values, observed, orders, batch=16):
model.eval()
output = []
with torch.no_grad():
for start in range(0, len(values), batch):
x, m = torch.tensor(values[start:start+batch]), torch.tensor(observed[start:start+batch])
z = torch.stack([model.encode(x, m, torch.tensor(order, dtype=torch.long)) for order in orders]).mean(0)
output.append(z.numpy())
return np.concatenate(output)
class LatentPredictor:
def __init__(self, bundle):
bundle = Path(bundle)
self.spec = json.loads((bundle/"model.json").read_text())
with np.load(bundle/"preprocessor.npz", allow_pickle=False) as archive:
self.preprocessor = dict(archive)
self.model = ProteinAR(len(self.spec["proteins"]), **self.spec["config"])
self.model.load_state_dict(torch.load(bundle/"weights.pt", map_location="cpu", weights_only=True))
self.model.eval()
def predict(self, events):
for event in events:
if set(event) != {"visit_id", "visit_month", "proteins"}:
raise ValueError("Only current visit metadata and proteins are accepted")
if len(event["proteins"]) != len(self.spec["proteins"]):
raise ValueError("Protein vocabulary mismatch")
raw = np.asarray([e["proteins"] for e in events], dtype=float)
values, mask = preprocess(raw, self.preprocessor)
return encode_values(self.model, values, mask, self.spec["orders"])
|