import copy import json from pathlib import Path import numpy as np import torch import torch.nn as nn from transformers import EsmModel class ESMClassifier(nn.Module): def __init__(self, esm_path: str, num_classes: int, dropout: float = 0.1, sdpa: bool = True): super().__init__() kwargs = dict(add_pooling_layer=False) if sdpa: try: self.esm = EsmModel.from_pretrained(esm_path, attn_implementation="sdpa", **kwargs) except (ValueError, TypeError, NotImplementedError): self.esm = EsmModel.from_pretrained(esm_path, **kwargs) else: self.esm = EsmModel.from_pretrained(esm_path, **kwargs) H = self.esm.config.hidden_size self.classifier = nn.Sequential( nn.Dropout(dropout), nn.Linear(H, H), nn.GELU(), nn.Dropout(dropout), nn.Linear(H, num_classes), ) def forward(self, input_ids, attention_mask): hidden = self.esm(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state if attention_mask is None: pooled = hidden.mean(1) else: mask = attention_mask.unsqueeze(-1).to(hidden.dtype) pooled = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1) return self.classifier(pooled) class BF16Model(nn.Module): def __init__(self, model): super().__init__() self.model = model def forward(self, input_ids, attention_mask): with torch.autocast("cpu", dtype=torch.bfloat16): return self.model(input_ids, attention_mask) class OrtVariant: """ONNX Runtime backbone + the torch pooling/head (same maths as ESMClassifier.forward). Needs weights/onnx/esm_fp32.onnx or esm_int8.onnx (made by export_onnx.py) and `onnxruntime`. """ def __init__(self, onnx_path, classifier: nn.Module, threads: int = 1): import onnxruntime as ort so = ort.SessionOptions() so.intra_op_num_threads = threads so.inter_op_num_threads = 1 so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL self.sess = ort.InferenceSession(str(onnx_path), so, providers=["CPUExecutionProvider"]) self.classifier = classifier def __call__(self, input_ids, attention_mask): ids = input_ids.numpy().astype(np.int64) m = np.ones_like(ids) if attention_mask is None else attention_mask.numpy().astype(np.int64) hidden = torch.from_numpy(self.sess.run(None, {"input_ids": ids, "attention_mask": m})[0]) if attention_mask is None: pooled = hidden.mean(1) else: mk = attention_mask.unsqueeze(-1).to(hidden.dtype) pooled = (hidden * mk).sum(1) / mk.sum(1).clamp(min=1) return self.classifier(pooled) def load_model(weights_dir: Path, sdpa: bool = True): weights_dir = Path(weights_dir) meta = json.loads((weights_dir / "meta.json").read_text()) classes = json.loads((weights_dir / "classes.json").read_text()) model = ESMClassifier( esm_path = str(weights_dir / "esm2"), num_classes = meta["num_classes"], dropout = meta.get("dropout", 0.1), sdpa = sdpa, ) head_state = torch.load(weights_dir / "classifier_head.pt", map_location="cpu", weights_only=True) model.classifier.load_state_dict(head_state) model.eval() return model, meta, classes def restrict_classes(model: ESMClassifier, keep): old = model.classifier[-1] new = nn.Linear(old.in_features, len(keep)) with torch.no_grad(): new.weight.copy_(old.weight[keep]) new.bias.copy_(old.bias[keep]) model.classifier[-1] = new model.eval() return model # --------------------------------------------------------------------------- # Fused Q/K/V projection: one GEMM (and one int8 activation quantisation) instead of three. # Mathematically identical; verified numerically against the original before use. # --------------------------------------------------------------------------- class _QKVHub(nn.Module): def __init__(self, q: nn.Linear, k: nn.Linear, v: nn.Linear): super().__init__() self.H = q.out_features self.fused = nn.Linear(q.in_features, 3 * self.H, bias=q.bias is not None) with torch.no_grad(): self.fused.weight.copy_(torch.cat([q.weight, k.weight, v.weight], 0)) if q.bias is not None: self.fused.bias.copy_(torch.cat([q.bias, k.bias, v.bias], 0)) self._x, self._out, self._seen = None, None, set() def project(self, x, which): if self._out is None or self._x is not x: self._out = self.fused(x).split(self.H, dim=-1) self._x, self._seen = x, set() out = self._out[which] self._seen.add(which) if len(self._seen) == 3: self._x, self._out, self._seen = None, None, set() return out class _QKVProxy(nn.Module): def __init__(self, hub, which): super().__init__() self._hub = [hub] self.which = which def forward(self, x): return self._hub[0].project(x, self.which) def fuse_qkv(model: ESMClassifier) -> int: n = 0 for layer in model.esm.encoder.layer: sa = layer.attention.self q, k, v = sa.query, sa.key, sa.value if not all(isinstance(t, nn.Linear) for t in (q, k, v)): continue hub = _QKVHub(q, k, v) sa.qkv = hub sa.query, sa.key, sa.value = _QKVProxy(hub, 0), _QKVProxy(hub, 1), _QKVProxy(hub, 2) n += 1 return n def fuse_qkv_verified(model: ESMClassifier, input_ids, attention_mask, tol: float = 5e-3): """Returns (model, n_fused). Falls back to the untouched model if anything differs.""" ref = copy.deepcopy(model) try: n = fuse_qkv(model) if n == 0: return ref, 0 with torch.inference_mode(): a = ref(input_ids, attention_mask).float() b = model(input_ids, attention_mask).float() if torch.isfinite(b).all() and (a - b).abs().max().item() < tol: return model, n except Exception: pass return ref, 0 def int8_copy(model: ESMClassifier, ffn_only: bool = False, per_channel: bool = True): """Dynamic int8 copy of the ESM encoder linears (classifier head stays fp32). Per-channel weight scales cost the same at inference as per-tensor scales but are noticeably closer to the fp32 outputs, so accuracy loss from int8 shrinks. """ names = set() for name, m in model.named_modules(): if not isinstance(m, nn.Linear) or not name.startswith("esm."): continue if ffn_only and ("attention" in name or not ("intermediate" in name or "output" in name)): continue names.add(name) q = torch.ao.quantization if per_channel: try: spec = {n: q.per_channel_dynamic_qconfig for n in names} return q.quantize_dynamic(model, spec, dtype=torch.qint8) except Exception: pass return q.quantize_dynamic(model, names, dtype=torch.qint8) def build_variants(model: ESMClassifier, kinds, ffn_only: bool = False, onnx_dir=None, threads: int = 1): known = {"fp32", "bf16", "int8", "ort_fp32", "ort_int8"} variants = {} for kind in kinds: if kind not in known: raise ValueError(f"unknown precision {kind}") try: if kind == "fp32": variants[kind] = model elif kind == "bf16": variants[kind] = BF16Model(model) elif kind == "int8": variants[kind] = int8_copy(model, ffn_only) else: variants[kind] = OrtVariant(Path(onnx_dir) / f"esm_{kind[4:]}.onnx", model.classifier, threads) except Exception as e: print(f"[build_variants] {kind} unavailable: {e!r}", flush=True) return variants