ec-esm13 / predict.py
sahithimacharapu2's picture
Upload ESM-2 35M enzyme EC classifier
b2f7026 verified
Raw History Blame Contribute Delete
17 kB
import argparse
import json
import math
import os
import queue
import time
import traceback
from pathlib import Path
import numpy as np
import pandas as pd
import torch
from transformers import AutoTokenizer
from model import load_model, restrict_classes, build_variants, fuse_qkv_verified
from inputs import prepare_inputs
T0 = time.time()
ROOT = Path(__file__).parent
WEIGHTS = ROOT / "weights"
TIME_BUDGET = float(os.environ.get("EC_TIME_BUDGET", 660))
BATCH = int(os.environ.get("EC_BATCH", 64)) # max rows per batch
TOK_BUDGET = int(os.environ.get("EC_TOK_BUDGET", 2048)) # max tokens per batch (cache friendly)
THRESH = float(os.environ.get("EC_THRESH", 0.5)) # softmax + 0.5 => top-1 only (what training optimised)
MAX_EXTRA = 2
PRECISION = os.environ.get("EC_PRECISION", "auto")
ACC_TOL = float(os.environ.get("EC_ACC_TOL", 1.12))
QUANT_FFN = int(os.environ.get("EC_QUANT_FFN_ONLY", 0))
SDPA = int(os.environ.get("EC_SDPA", 1))
FUSE_QKV = int(os.environ.get("EC_FUSE_QKV", 1))
USE_ORT = int(os.environ.get("EC_ORT", 1)) # use weights/onnx/*.onnx when present
ONNX_DIR = WEIGHTS / "onnx"
MIN_LEN = int(os.environ.get("EC_MIN_LEN", 64))
SAFETY = float(os.environ.get("EC_SAFETY", 8))
PLAN_MARGIN = float(os.environ.get("EC_PLAN_MARGIN", 0.96))
WORKERS = int(os.environ.get("EC_WORKERS", 0))
THREADS = int(os.environ.get("EC_THREADS", 0))
WORKER_GB = float(os.environ.get("EC_WORKER_GB", 1.2))
BENCH_N = int(os.environ.get("EC_BENCH_N", 8))
BENCH_LEN = int(os.environ.get("EC_BENCH_LEN", 160))
L0 = float(os.environ.get("EC_L0", 800)) # cost(len) ~ len * (1 + len / L0)
MAX_CAP = int(os.environ.get("EC_MAX_CAP", 0)) # experiments only: force a length cap (0 = off)
LOOKUP = int(os.environ.get("EC_LOOKUP", 1))
LK_SKIP = float(os.environ.get("EC_LOOKUP_SKIP", 0.6)) # score >= this: take the training-set hit, skip the transformer
LK_MID = float(os.environ.get("EC_LOOKUP_MID", 0.15)) # score >= this AND model confidence < EC_CONF: take the hit
CONF = float(os.environ.get("EC_CONF", 0.6))
def log(msg: str, tag: str = ""):
print(f"[{time.time() - T0:6.1f}s]{tag} {msg}", flush=True)
def n_cpus() -> int:
try:
n = len(os.sched_getaffinity(0))
except AttributeError:
n = os.cpu_count() or 2
quota = None
try:
q, p = Path("/sys/fs/cgroup/cpu.max").read_text().split()
if q != "max":
quota = int(q) / int(p)
except Exception:
try:
q = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read_text())
p = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read_text())
if q > 0:
quota = q / p
except Exception:
pass
log(f"cpu affinity={n}, cgroup quota={quota}")
if quota:
n = min(n, max(1, math.ceil(quota)))
return n
def mem_gb() -> float:
vals = []
try:
for line in Path("/proc/meminfo").read_text().splitlines():
if line.startswith("MemAvailable"):
vals.append(int(line.split()[1]) / 1e6)
except Exception:
pass
for p in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
try:
v = Path(p).read_text().strip()
if v.isdigit():
vals.append(int(v) / 1e9)
except Exception:
pass
return min(vals) if vals else 1e9
def cpu_flags() -> set:
try:
for line in Path("/proc/cpuinfo").read_text().splitlines():
if line.startswith("flags"):
return set(line.split(":", 1)[1].split())
except Exception:
pass
return set()
def candidate_precisions():
"""Ordered most-accurate first; the fastest kind wins unless an earlier one is within ACC_TOL of it."""
if PRECISION != "auto":
return [PRECISION]
ort_ok = False
if USE_ORT and ONNX_DIR.exists():
try:
import onnxruntime # noqa: F401
ort_ok = True
except Exception:
pass
kinds = ["fp32"]
if ort_ok and (ONNX_DIR / "esm_fp32.onnx").exists():
kinds.append("ort_fp32")
if cpu_flags() & {"amx_bf16", "avx512_bf16"}:
kinds.append("bf16")
kinds.append("int8")
if ort_ok and (ONNX_DIR / "esm_int8.onnx").exists():
kinds.append("ort_int8")
return kinds
def cost_units(lens, cap):
e = np.minimum(lens, cap)
return float((e * (1.0 + e / L0)).sum())
def choose_cap(rem_lens, norm_rate, time_left, max_len):
"""Largest cap such that the remaining rows fit in the time left (length-aware cost model)."""
budget = max(0.0, time_left) * PLAN_MARGIN * norm_rate
lo, hi = MIN_LEN, max_len
if cost_units(rem_lens, hi) <= budget:
return hi
while lo < hi:
mid = (lo + hi + 1) // 2
if cost_units(rem_lens, mid) <= budget:
lo = mid
else:
hi = mid - 1
return lo
def worker(wid, seqs, lens, t0_abs, threads, cursor, rates_arr, barrier, out_q):
"""All workers pull batches from ONE shared length-sorted queue (perfect load balance, one global cap).
Batches are built from rows of IDENTICAL (capped) length whenever possible: zero padding,
no attention mask needed -> fastest attention path, and no wasted compute.
"""
global T0
T0 = t0_abs
tag = f" [w{wid}]"
done = {}
try:
torch.set_num_threads(threads)
try:
torch.set_num_interop_threads(1)
except RuntimeError:
pass
torch.set_flush_denormal(True)
tok = AutoTokenizer.from_pretrained(str(WEIGHTS / "esm2"))
n = len(seqs)
k = 1 + MAX_EXTRA
deadline = t0_abs + TIME_BUDGET - SAFETY
vocab = set(json.loads((ROOT / "ec_vocabulary.json").read_text()))
def run(idx, length, fwd, l4):
t = time.time()
enc = tok([seqs[i] for i in idx], padding="longest", truncation=True,
max_length=length, return_tensors="pt")
mask = enc["attention_mask"]
if bool(mask.all()): # equal lengths: no padding, no mask
mask = None
with torch.inference_mode():
logits = fwd(enc["input_ids"], mask).float()
probs = torch.softmax(logits, dim=-1)
top_p, top_j = probs.topk(min(k, probs.shape[1]), dim=1)
top_p, top_j = top_p.tolist(), top_j.tolist()
for row, i in enumerate(idx):
best = [l4[top_j[row][0]]]
if THRESH <= 1.0:
for p, j in zip(top_p[row][1:], top_j[row][1:]):
if p >= THRESH:
best.append(l4[j])
done[i] = (best, top_p[row][0])
return enc["input_ids"].numel(), time.time() - t
start = min(int(0.6 * n), max(0, n - BENCH_N))
probe = list(range(start, min(n, start + BENCH_N)))
kinds = candidate_precisions()
# ---- build + benchmark; if the fused-QKV int8 path misbehaves, redo without fusion
for fuse in ((True, False) if FUSE_QKV else (False,)):
model, meta, classes = load_model(WEIGHTS, sdpa=bool(SDPA))
max_len = meta["max_len"]
l4 = classes["l4"]
keep = [i for i, ec in enumerate(l4) if ec in vocab]
if not keep:
raise RuntimeError("No predicted class is in ec_vocabulary.json")
l4 = [l4[i] for i in keep]
restrict_classes(model, keep)
n_fused = 0
if fuse:
enc = tok([seqs[i] for i in probe[:3]], padding="longest", truncation=True,
max_length=96, return_tensors="pt")
model, n_fused = fuse_qkv_verified(model, enc["input_ids"], enc["attention_mask"])
variants = build_variants(model, kinds, bool(QUANT_FFN), ONNX_DIR, threads)
rates, ref = {}, None
def logits_of(idx, length, f):
enc_ = tok([seqs[i] for i in idx], padding="longest", truncation=True,
max_length=length, return_tensors="pt")
with torch.inference_mode():
return f(enc_["input_ids"], enc_["attention_mask"]).float()
for kind in kinds:
try:
if kind.startswith("ort") and "fp32" in variants: # parity check vs torch fp32
pidx = probe + [0] # row 0 is the shortest -> the batch is padded
if ref is None:
ref = logits_of(pidx, 128, variants["fp32"])
out = logits_of(pidx, 128, variants[kind])
rel = ((out - ref).norm() / ref.norm()).item()
if not rel < (0.02 if kind == "ort_fp32" else 0.35):
log(f"{kind} rejected: relative logit error {rel:.3f}", tag)
continue
run(probe[:2], 64, variants[kind], l4)
toks, secs = run(probe, min(BENCH_LEN, max_len), variants[kind], l4)
rates[kind] = toks / secs
except Exception as e:
log(f"{kind} failed during benchmark (fused={n_fused}): {e!r}", tag)
if rates and ("int8" in rates or "int8" not in kinds or not n_fused):
break
done.clear()
best = max(rates.values())
choice = next(kind for kind in kinds if kind in rates and rates[kind] >= best / ACC_TOL)
fwd = variants[choice]
variants = model = None
rates_arr[wid] = rates[choice] * 0.85 * (1 + min(BENCH_LEN, max_len) / L0)
if wid == 0:
log(f"rates tok/s { {k_: round(v) for k_, v in rates.items()} } -> {choice}, "
f"threads={threads}, fused_qkv_layers={n_fused}", tag)
try:
barrier.wait(timeout=90)
except Exception:
pass
def claim():
with cursor.get_lock():
c = cursor.value
now = time.time()
if c >= n or now > deadline:
return None
cap = choose_cap(lens[c:], sum(rates_arr[:]), deadline - now,
min(max_len, MAX_CAP) if MAX_CAP else max_len)
L, e = min(lens[c], cap), c + 1
while e < n and e - c < BATCH and min(lens[e], cap) == L and L * (e + 1 - c) <= TOK_BUDGET:
e += 1
if e - c < 3: # sparse length: allow a few near-equal lengths (uses the mask path)
while e < n and e - c < BATCH and min(lens[e], cap) - L <= 8 \
and min(lens[e], cap) * (e + 1 - c) <= TOK_BUDGET:
e += 1
cursor.value = e
return c, e, cap
capped_rows = nb = 0
cap, last_log = max_len, time.time()
while True:
job = claim()
if job is None:
break
c, e, cap = job
capped_rows += int((lens[c:e] > cap).sum())
toks, secs = run(list(range(c, e)), cap, fwd, l4)
if toks >= 1024 and secs > 0:
mean_len = toks / (e - c)
rates_arr[wid] = 0.7 * rates_arr[wid] + 0.3 * (toks / secs) * (1 + mean_len / L0)
nb += 1
if wid == 0 and time.time() - last_log > 45:
last_log = time.time()
log(f"row {c}/{n} cap={cap} agg norm rate={sum(rates_arr[:]):.0f}", tag)
log(f"finished: {len(done)} rows, {nb} batches, last cap={cap}, truncated rows={capped_rows}", tag)
except Exception:
log("worker crashed:\n" + traceback.format_exc(), tag)
finally:
out_q.put((wid, list(done.items())))
def predict_file(input_path: str, output_path: str):
frame = prepare_inputs(pd.read_parquet(input_path))
n = len(frame)
log(f"{n} unique inputs")
classes = json.loads((WEIGHTS / "classes.json").read_text())
fallback = classes["fallback_ec"]
seqs = [str(s).upper() for s in frame.sequence]
lens = np.array([len(s) + 2 for s in seqs])
if n:
log(f"token length p50/p90/p99 = {np.percentile(lens, [50, 90, 99]).astype(int).tolist()}, "
f"mean={lens.mean():.0f}")
preds = [[fallback]] * n
lk_ec, lk_sc = [None] * n, np.zeros(n)
todo = np.arange(n)
lk_dir = WEIGHTS / "lookup"
if n and LOOKUP and (lk_dir / "meta.json").exists():
try:
from lookup import Lookup
lk = Lookup(lk_dir)
for i, s in enumerate(seqs):
lk_ec[i], lk_sc[i] = lk.query(s)
stats = {t: f"{(lk_sc >= t).mean():.1%}" for t in (0.06, 0.15, 0.3, 0.6, 0.9)}
log(f"lookup done: share of rows with score >= t: {stats}")
skip = lk_sc >= LK_SKIP
for i in np.flatnonzero(skip):
preds[i] = [lk_ec[i]]
todo = np.flatnonzero(~skip)
log(f"{int(skip.sum())} rows resolved by lookup, {len(todo)} go to the transformer")
except Exception:
log("lookup failed, continuing without it:\n" + traceback.format_exc())
lk_ec, lk_sc, todo = [None] * n, np.zeros(n), np.arange(n)
if len(todo):
cpus = n_cpus()
workers = WORKERS or cpus
workers = max(1, min(workers, cpus, len(todo), int(mem_gb() / WORKER_GB)))
threads = THREADS or max(1, cpus // workers)
log(f"cpus={cpus} -> workers={workers} x threads={threads}")
order = todo[np.argsort(lens[todo], kind="stable")] # global row ids, ascending length
sorted_seqs = [seqs[i] for i in order]
sorted_lens = lens[order]
results = []
import multiprocessing as mp
ctx = mp.get_context("spawn")
if workers > 1:
procs = []
try:
cursor = ctx.Value("i", 0)
rates = ctx.Array("d", workers)
barrier = ctx.Barrier(workers)
out_q = ctx.Queue()
procs = [ctx.Process(target=worker, args=(w, sorted_seqs, sorted_lens, T0, threads,
cursor, rates, barrier, out_q))
for w in range(workers)]
for p_ in procs:
p_.start()
pending, hard_stop = workers, T0 + TIME_BUDGET - 1
while pending and time.time() < hard_stop:
try:
results.append(out_q.get(timeout=2))
pending -= 1
except queue.Empty:
if not any(p_.is_alive() for p_ in procs):
while True:
try:
results.append(out_q.get(timeout=1))
except queue.Empty:
break
break
for p_ in procs:
if p_.is_alive():
p_.terminate()
except Exception as e:
log(f"multiprocessing failed ({e!r}); falling back to a single process")
for p_ in procs:
if p_.is_alive():
p_.terminate()
results = []
if not results:
workers = 1
if workers == 1:
q1 = queue.Queue()
worker(0, sorted_seqs, sorted_lens, T0, cpus, ctx.Value("i", 0), ctx.Array("d", 1),
ctx.Barrier(1), q1)
results = [q1.get()]
got = swapped = 0
for _, part in results:
for s, (best, conf) in part:
i = order[s]
if lk_ec[i] is not None and lk_sc[i] >= LK_MID and conf < CONF:
preds[i] = [lk_ec[i]]
swapped += 1
else:
preds[i] = best
got += 1
log(f"transformer predicted {got}/{len(todo)} rows ({len(todo) - got} use fallback EC); "
f"{swapped} low-confidence rows replaced by lookup hit")
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
with Path(output_path).open("w") as f:
for key, p in zip(frame.id, preds):
f.write(json.dumps({"id": key, "prediction": p}) + "\n")
log("done")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input", required=True, help="Input Parquet file")
parser.add_argument("--output", required=True, help="Output JSONL file")
args = parser.parse_args()
predict_file(args.input, args.output)