ec-esm13 / model.py
sahithimacharapu2's picture
Upload ESM-2 35M enzyme EC classifier
b2f7026 verified
Raw History Blame Contribute Delete
8.08 kB
import copy
import json
from pathlib import Path
import numpy as np
import torch
import torch.nn as nn
from transformers import EsmModel
class ESMClassifier(nn.Module):
def __init__(self, esm_path: str, num_classes: int, dropout: float = 0.1, sdpa: bool = True):
super().__init__()
kwargs = dict(add_pooling_layer=False)
if sdpa:
try:
self.esm = EsmModel.from_pretrained(esm_path, attn_implementation="sdpa", **kwargs)
except (ValueError, TypeError, NotImplementedError):
self.esm = EsmModel.from_pretrained(esm_path, **kwargs)
else:
self.esm = EsmModel.from_pretrained(esm_path, **kwargs)
H = self.esm.config.hidden_size
self.classifier = nn.Sequential(
nn.Dropout(dropout),
nn.Linear(H, H),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(H, num_classes),
)
def forward(self, input_ids, attention_mask):
hidden = self.esm(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
if attention_mask is None:
pooled = hidden.mean(1)
else:
mask = attention_mask.unsqueeze(-1).to(hidden.dtype)
pooled = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1)
return self.classifier(pooled)
class BF16Model(nn.Module):
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, input_ids, attention_mask):
with torch.autocast("cpu", dtype=torch.bfloat16):
return self.model(input_ids, attention_mask)
class OrtVariant:
"""ONNX Runtime backbone + the torch pooling/head (same maths as ESMClassifier.forward).
Needs weights/onnx/esm_fp32.onnx or esm_int8.onnx (made by export_onnx.py) and `onnxruntime`.
"""
def __init__(self, onnx_path, classifier: nn.Module, threads: int = 1):
import onnxruntime as ort
so = ort.SessionOptions()
so.intra_op_num_threads = threads
so.inter_op_num_threads = 1
so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
self.sess = ort.InferenceSession(str(onnx_path), so, providers=["CPUExecutionProvider"])
self.classifier = classifier
def __call__(self, input_ids, attention_mask):
ids = input_ids.numpy().astype(np.int64)
m = np.ones_like(ids) if attention_mask is None else attention_mask.numpy().astype(np.int64)
hidden = torch.from_numpy(self.sess.run(None, {"input_ids": ids, "attention_mask": m})[0])
if attention_mask is None:
pooled = hidden.mean(1)
else:
mk = attention_mask.unsqueeze(-1).to(hidden.dtype)
pooled = (hidden * mk).sum(1) / mk.sum(1).clamp(min=1)
return self.classifier(pooled)
def load_model(weights_dir: Path, sdpa: bool = True):
weights_dir = Path(weights_dir)
meta = json.loads((weights_dir / "meta.json").read_text())
classes = json.loads((weights_dir / "classes.json").read_text())
model = ESMClassifier(
esm_path = str(weights_dir / "esm2"),
num_classes = meta["num_classes"],
dropout = meta.get("dropout", 0.1),
sdpa = sdpa,
)
head_state = torch.load(weights_dir / "classifier_head.pt", map_location="cpu", weights_only=True)
model.classifier.load_state_dict(head_state)
model.eval()
return model, meta, classes
def restrict_classes(model: ESMClassifier, keep):
old = model.classifier[-1]
new = nn.Linear(old.in_features, len(keep))
with torch.no_grad():
new.weight.copy_(old.weight[keep])
new.bias.copy_(old.bias[keep])
model.classifier[-1] = new
model.eval()
return model
# ---------------------------------------------------------------------------
# Fused Q/K/V projection: one GEMM (and one int8 activation quantisation) instead of three.
# Mathematically identical; verified numerically against the original before use.
# ---------------------------------------------------------------------------
class _QKVHub(nn.Module):
def __init__(self, q: nn.Linear, k: nn.Linear, v: nn.Linear):
super().__init__()
self.H = q.out_features
self.fused = nn.Linear(q.in_features, 3 * self.H, bias=q.bias is not None)
with torch.no_grad():
self.fused.weight.copy_(torch.cat([q.weight, k.weight, v.weight], 0))
if q.bias is not None:
self.fused.bias.copy_(torch.cat([q.bias, k.bias, v.bias], 0))
self._x, self._out, self._seen = None, None, set()
def project(self, x, which):
if self._out is None or self._x is not x:
self._out = self.fused(x).split(self.H, dim=-1)
self._x, self._seen = x, set()
out = self._out[which]
self._seen.add(which)
if len(self._seen) == 3:
self._x, self._out, self._seen = None, None, set()
return out
class _QKVProxy(nn.Module):
def __init__(self, hub, which):
super().__init__()
self._hub = [hub]
self.which = which
def forward(self, x):
return self._hub[0].project(x, self.which)
def fuse_qkv(model: ESMClassifier) -> int:
n = 0
for layer in model.esm.encoder.layer:
sa = layer.attention.self
q, k, v = sa.query, sa.key, sa.value
if not all(isinstance(t, nn.Linear) for t in (q, k, v)):
continue
hub = _QKVHub(q, k, v)
sa.qkv = hub
sa.query, sa.key, sa.value = _QKVProxy(hub, 0), _QKVProxy(hub, 1), _QKVProxy(hub, 2)
n += 1
return n
def fuse_qkv_verified(model: ESMClassifier, input_ids, attention_mask, tol: float = 5e-3):
"""Returns (model, n_fused). Falls back to the untouched model if anything differs."""
ref = copy.deepcopy(model)
try:
n = fuse_qkv(model)
if n == 0:
return ref, 0
with torch.inference_mode():
a = ref(input_ids, attention_mask).float()
b = model(input_ids, attention_mask).float()
if torch.isfinite(b).all() and (a - b).abs().max().item() < tol:
return model, n
except Exception:
pass
return ref, 0
def int8_copy(model: ESMClassifier, ffn_only: bool = False, per_channel: bool = True):
"""Dynamic int8 copy of the ESM encoder linears (classifier head stays fp32).
Per-channel weight scales cost the same at inference as per-tensor scales
but are noticeably closer to the fp32 outputs, so accuracy loss from int8 shrinks.
"""
names = set()
for name, m in model.named_modules():
if not isinstance(m, nn.Linear) or not name.startswith("esm."):
continue
if ffn_only and ("attention" in name or not ("intermediate" in name or "output" in name)):
continue
names.add(name)
q = torch.ao.quantization
if per_channel:
try:
spec = {n: q.per_channel_dynamic_qconfig for n in names}
return q.quantize_dynamic(model, spec, dtype=torch.qint8)
except Exception:
pass
return q.quantize_dynamic(model, names, dtype=torch.qint8)
def build_variants(model: ESMClassifier, kinds, ffn_only: bool = False, onnx_dir=None, threads: int = 1):
known = {"fp32", "bf16", "int8", "ort_fp32", "ort_int8"}
variants = {}
for kind in kinds:
if kind not in known:
raise ValueError(f"unknown precision {kind}")
try:
if kind == "fp32":
variants[kind] = model
elif kind == "bf16":
variants[kind] = BF16Model(model)
elif kind == "int8":
variants[kind] = int8_copy(model, ffn_only)
else:
variants[kind] = OrtVariant(Path(onnx_dir) / f"esm_{kind[4:]}.onnx", model.classifier, threads)
except Exception as e:
print(f"[build_variants] {kind} unavailable: {e!r}", flush=True)
return variants