Instructions to use sahithimacharapu2/ec-esm13 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use sahithimacharapu2/ec-esm13 with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("sahithimacharapu2/ec-esm13", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download predict.py from sahithimacharapu2/ec-esm13: direct link, hf CLI and curl.
- Browser
- Download file 17 kB
-
https://huggingface.co/sahithimacharapu2/ec-esm13/resolve/main/predict.py
- Command line
-
hf download hf://sahithimacharapu2/ec-esm13/predict.py
-
curl -L -o predict.py https://huggingface.co/sahithimacharapu2/ec-esm13/resolve/main/predict.py
17 kB
| import argparse | |
| import json | |
| import math | |
| import os | |
| import queue | |
| import time | |
| import traceback | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from transformers import AutoTokenizer | |
| from model import load_model, restrict_classes, build_variants, fuse_qkv_verified | |
| from inputs import prepare_inputs | |
| T0 = time.time() | |
| ROOT = Path(__file__).parent | |
| WEIGHTS = ROOT / "weights" | |
| TIME_BUDGET = float(os.environ.get("EC_TIME_BUDGET", 660)) | |
| BATCH = int(os.environ.get("EC_BATCH", 64)) # max rows per batch | |
| TOK_BUDGET = int(os.environ.get("EC_TOK_BUDGET", 2048)) # max tokens per batch (cache friendly) | |
| THRESH = float(os.environ.get("EC_THRESH", 0.5)) # softmax + 0.5 => top-1 only (what training optimised) | |
| MAX_EXTRA = 2 | |
| PRECISION = os.environ.get("EC_PRECISION", "auto") | |
| ACC_TOL = float(os.environ.get("EC_ACC_TOL", 1.12)) | |
| QUANT_FFN = int(os.environ.get("EC_QUANT_FFN_ONLY", 0)) | |
| SDPA = int(os.environ.get("EC_SDPA", 1)) | |
| FUSE_QKV = int(os.environ.get("EC_FUSE_QKV", 1)) | |
| USE_ORT = int(os.environ.get("EC_ORT", 1)) # use weights/onnx/*.onnx when present | |
| ONNX_DIR = WEIGHTS / "onnx" | |
| MIN_LEN = int(os.environ.get("EC_MIN_LEN", 64)) | |
| SAFETY = float(os.environ.get("EC_SAFETY", 8)) | |
| PLAN_MARGIN = float(os.environ.get("EC_PLAN_MARGIN", 0.96)) | |
| WORKERS = int(os.environ.get("EC_WORKERS", 0)) | |
| THREADS = int(os.environ.get("EC_THREADS", 0)) | |
| WORKER_GB = float(os.environ.get("EC_WORKER_GB", 1.2)) | |
| BENCH_N = int(os.environ.get("EC_BENCH_N", 8)) | |
| BENCH_LEN = int(os.environ.get("EC_BENCH_LEN", 160)) | |
| L0 = float(os.environ.get("EC_L0", 800)) # cost(len) ~ len * (1 + len / L0) | |
| MAX_CAP = int(os.environ.get("EC_MAX_CAP", 0)) # experiments only: force a length cap (0 = off) | |
| LOOKUP = int(os.environ.get("EC_LOOKUP", 1)) | |
| LK_SKIP = float(os.environ.get("EC_LOOKUP_SKIP", 0.6)) # score >= this: take the training-set hit, skip the transformer | |
| LK_MID = float(os.environ.get("EC_LOOKUP_MID", 0.15)) # score >= this AND model confidence < EC_CONF: take the hit | |
| CONF = float(os.environ.get("EC_CONF", 0.6)) | |
| def log(msg: str, tag: str = ""): | |
| print(f"[{time.time() - T0:6.1f}s]{tag} {msg}", flush=True) | |
| def n_cpus() -> int: | |
| try: | |
| n = len(os.sched_getaffinity(0)) | |
| except AttributeError: | |
| n = os.cpu_count() or 2 | |
| quota = None | |
| try: | |
| q, p = Path("/sys/fs/cgroup/cpu.max").read_text().split() | |
| if q != "max": | |
| quota = int(q) / int(p) | |
| except Exception: | |
| try: | |
| q = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read_text()) | |
| p = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read_text()) | |
| if q > 0: | |
| quota = q / p | |
| except Exception: | |
| pass | |
| log(f"cpu affinity={n}, cgroup quota={quota}") | |
| if quota: | |
| n = min(n, max(1, math.ceil(quota))) | |
| return n | |
| def mem_gb() -> float: | |
| vals = [] | |
| try: | |
| for line in Path("/proc/meminfo").read_text().splitlines(): | |
| if line.startswith("MemAvailable"): | |
| vals.append(int(line.split()[1]) / 1e6) | |
| except Exception: | |
| pass | |
| for p in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"): | |
| try: | |
| v = Path(p).read_text().strip() | |
| if v.isdigit(): | |
| vals.append(int(v) / 1e9) | |
| except Exception: | |
| pass | |
| return min(vals) if vals else 1e9 | |
| def cpu_flags() -> set: | |
| try: | |
| for line in Path("/proc/cpuinfo").read_text().splitlines(): | |
| if line.startswith("flags"): | |
| return set(line.split(":", 1)[1].split()) | |
| except Exception: | |
| pass | |
| return set() | |
| def candidate_precisions(): | |
| """Ordered most-accurate first; the fastest kind wins unless an earlier one is within ACC_TOL of it.""" | |
| if PRECISION != "auto": | |
| return [PRECISION] | |
| ort_ok = False | |
| if USE_ORT and ONNX_DIR.exists(): | |
| try: | |
| import onnxruntime # noqa: F401 | |
| ort_ok = True | |
| except Exception: | |
| pass | |
| kinds = ["fp32"] | |
| if ort_ok and (ONNX_DIR / "esm_fp32.onnx").exists(): | |
| kinds.append("ort_fp32") | |
| if cpu_flags() & {"amx_bf16", "avx512_bf16"}: | |
| kinds.append("bf16") | |
| kinds.append("int8") | |
| if ort_ok and (ONNX_DIR / "esm_int8.onnx").exists(): | |
| kinds.append("ort_int8") | |
| return kinds | |
| def cost_units(lens, cap): | |
| e = np.minimum(lens, cap) | |
| return float((e * (1.0 + e / L0)).sum()) | |
| def choose_cap(rem_lens, norm_rate, time_left, max_len): | |
| """Largest cap such that the remaining rows fit in the time left (length-aware cost model).""" | |
| budget = max(0.0, time_left) * PLAN_MARGIN * norm_rate | |
| lo, hi = MIN_LEN, max_len | |
| if cost_units(rem_lens, hi) <= budget: | |
| return hi | |
| while lo < hi: | |
| mid = (lo + hi + 1) // 2 | |
| if cost_units(rem_lens, mid) <= budget: | |
| lo = mid | |
| else: | |
| hi = mid - 1 | |
| return lo | |
| def worker(wid, seqs, lens, t0_abs, threads, cursor, rates_arr, barrier, out_q): | |
| """All workers pull batches from ONE shared length-sorted queue (perfect load balance, one global cap). | |
| Batches are built from rows of IDENTICAL (capped) length whenever possible: zero padding, | |
| no attention mask needed -> fastest attention path, and no wasted compute. | |
| """ | |
| global T0 | |
| T0 = t0_abs | |
| tag = f" [w{wid}]" | |
| done = {} | |
| try: | |
| torch.set_num_threads(threads) | |
| try: | |
| torch.set_num_interop_threads(1) | |
| except RuntimeError: | |
| pass | |
| torch.set_flush_denormal(True) | |
| tok = AutoTokenizer.from_pretrained(str(WEIGHTS / "esm2")) | |
| n = len(seqs) | |
| k = 1 + MAX_EXTRA | |
| deadline = t0_abs + TIME_BUDGET - SAFETY | |
| vocab = set(json.loads((ROOT / "ec_vocabulary.json").read_text())) | |
| def run(idx, length, fwd, l4): | |
| t = time.time() | |
| enc = tok([seqs[i] for i in idx], padding="longest", truncation=True, | |
| max_length=length, return_tensors="pt") | |
| mask = enc["attention_mask"] | |
| if bool(mask.all()): # equal lengths: no padding, no mask | |
| mask = None | |
| with torch.inference_mode(): | |
| logits = fwd(enc["input_ids"], mask).float() | |
| probs = torch.softmax(logits, dim=-1) | |
| top_p, top_j = probs.topk(min(k, probs.shape[1]), dim=1) | |
| top_p, top_j = top_p.tolist(), top_j.tolist() | |
| for row, i in enumerate(idx): | |
| best = [l4[top_j[row][0]]] | |
| if THRESH <= 1.0: | |
| for p, j in zip(top_p[row][1:], top_j[row][1:]): | |
| if p >= THRESH: | |
| best.append(l4[j]) | |
| done[i] = (best, top_p[row][0]) | |
| return enc["input_ids"].numel(), time.time() - t | |
| start = min(int(0.6 * n), max(0, n - BENCH_N)) | |
| probe = list(range(start, min(n, start + BENCH_N))) | |
| kinds = candidate_precisions() | |
| # ---- build + benchmark; if the fused-QKV int8 path misbehaves, redo without fusion | |
| for fuse in ((True, False) if FUSE_QKV else (False,)): | |
| model, meta, classes = load_model(WEIGHTS, sdpa=bool(SDPA)) | |
| max_len = meta["max_len"] | |
| l4 = classes["l4"] | |
| keep = [i for i, ec in enumerate(l4) if ec in vocab] | |
| if not keep: | |
| raise RuntimeError("No predicted class is in ec_vocabulary.json") | |
| l4 = [l4[i] for i in keep] | |
| restrict_classes(model, keep) | |
| n_fused = 0 | |
| if fuse: | |
| enc = tok([seqs[i] for i in probe[:3]], padding="longest", truncation=True, | |
| max_length=96, return_tensors="pt") | |
| model, n_fused = fuse_qkv_verified(model, enc["input_ids"], enc["attention_mask"]) | |
| variants = build_variants(model, kinds, bool(QUANT_FFN), ONNX_DIR, threads) | |
| rates, ref = {}, None | |
| def logits_of(idx, length, f): | |
| enc_ = tok([seqs[i] for i in idx], padding="longest", truncation=True, | |
| max_length=length, return_tensors="pt") | |
| with torch.inference_mode(): | |
| return f(enc_["input_ids"], enc_["attention_mask"]).float() | |
| for kind in kinds: | |
| try: | |
| if kind.startswith("ort") and "fp32" in variants: # parity check vs torch fp32 | |
| pidx = probe + [0] # row 0 is the shortest -> the batch is padded | |
| if ref is None: | |
| ref = logits_of(pidx, 128, variants["fp32"]) | |
| out = logits_of(pidx, 128, variants[kind]) | |
| rel = ((out - ref).norm() / ref.norm()).item() | |
| if not rel < (0.02 if kind == "ort_fp32" else 0.35): | |
| log(f"{kind} rejected: relative logit error {rel:.3f}", tag) | |
| continue | |
| run(probe[:2], 64, variants[kind], l4) | |
| toks, secs = run(probe, min(BENCH_LEN, max_len), variants[kind], l4) | |
| rates[kind] = toks / secs | |
| except Exception as e: | |
| log(f"{kind} failed during benchmark (fused={n_fused}): {e!r}", tag) | |
| if rates and ("int8" in rates or "int8" not in kinds or not n_fused): | |
| break | |
| done.clear() | |
| best = max(rates.values()) | |
| choice = next(kind for kind in kinds if kind in rates and rates[kind] >= best / ACC_TOL) | |
| fwd = variants[choice] | |
| variants = model = None | |
| rates_arr[wid] = rates[choice] * 0.85 * (1 + min(BENCH_LEN, max_len) / L0) | |
| if wid == 0: | |
| log(f"rates tok/s { {k_: round(v) for k_, v in rates.items()} } -> {choice}, " | |
| f"threads={threads}, fused_qkv_layers={n_fused}", tag) | |
| try: | |
| barrier.wait(timeout=90) | |
| except Exception: | |
| pass | |
| def claim(): | |
| with cursor.get_lock(): | |
| c = cursor.value | |
| now = time.time() | |
| if c >= n or now > deadline: | |
| return None | |
| cap = choose_cap(lens[c:], sum(rates_arr[:]), deadline - now, | |
| min(max_len, MAX_CAP) if MAX_CAP else max_len) | |
| L, e = min(lens[c], cap), c + 1 | |
| while e < n and e - c < BATCH and min(lens[e], cap) == L and L * (e + 1 - c) <= TOK_BUDGET: | |
| e += 1 | |
| if e - c < 3: # sparse length: allow a few near-equal lengths (uses the mask path) | |
| while e < n and e - c < BATCH and min(lens[e], cap) - L <= 8 \ | |
| and min(lens[e], cap) * (e + 1 - c) <= TOK_BUDGET: | |
| e += 1 | |
| cursor.value = e | |
| return c, e, cap | |
| capped_rows = nb = 0 | |
| cap, last_log = max_len, time.time() | |
| while True: | |
| job = claim() | |
| if job is None: | |
| break | |
| c, e, cap = job | |
| capped_rows += int((lens[c:e] > cap).sum()) | |
| toks, secs = run(list(range(c, e)), cap, fwd, l4) | |
| if toks >= 1024 and secs > 0: | |
| mean_len = toks / (e - c) | |
| rates_arr[wid] = 0.7 * rates_arr[wid] + 0.3 * (toks / secs) * (1 + mean_len / L0) | |
| nb += 1 | |
| if wid == 0 and time.time() - last_log > 45: | |
| last_log = time.time() | |
| log(f"row {c}/{n} cap={cap} agg norm rate={sum(rates_arr[:]):.0f}", tag) | |
| log(f"finished: {len(done)} rows, {nb} batches, last cap={cap}, truncated rows={capped_rows}", tag) | |
| except Exception: | |
| log("worker crashed:\n" + traceback.format_exc(), tag) | |
| finally: | |
| out_q.put((wid, list(done.items()))) | |
| def predict_file(input_path: str, output_path: str): | |
| frame = prepare_inputs(pd.read_parquet(input_path)) | |
| n = len(frame) | |
| log(f"{n} unique inputs") | |
| classes = json.loads((WEIGHTS / "classes.json").read_text()) | |
| fallback = classes["fallback_ec"] | |
| seqs = [str(s).upper() for s in frame.sequence] | |
| lens = np.array([len(s) + 2 for s in seqs]) | |
| if n: | |
| log(f"token length p50/p90/p99 = {np.percentile(lens, [50, 90, 99]).astype(int).tolist()}, " | |
| f"mean={lens.mean():.0f}") | |
| preds = [[fallback]] * n | |
| lk_ec, lk_sc = [None] * n, np.zeros(n) | |
| todo = np.arange(n) | |
| lk_dir = WEIGHTS / "lookup" | |
| if n and LOOKUP and (lk_dir / "meta.json").exists(): | |
| try: | |
| from lookup import Lookup | |
| lk = Lookup(lk_dir) | |
| for i, s in enumerate(seqs): | |
| lk_ec[i], lk_sc[i] = lk.query(s) | |
| stats = {t: f"{(lk_sc >= t).mean():.1%}" for t in (0.06, 0.15, 0.3, 0.6, 0.9)} | |
| log(f"lookup done: share of rows with score >= t: {stats}") | |
| skip = lk_sc >= LK_SKIP | |
| for i in np.flatnonzero(skip): | |
| preds[i] = [lk_ec[i]] | |
| todo = np.flatnonzero(~skip) | |
| log(f"{int(skip.sum())} rows resolved by lookup, {len(todo)} go to the transformer") | |
| except Exception: | |
| log("lookup failed, continuing without it:\n" + traceback.format_exc()) | |
| lk_ec, lk_sc, todo = [None] * n, np.zeros(n), np.arange(n) | |
| if len(todo): | |
| cpus = n_cpus() | |
| workers = WORKERS or cpus | |
| workers = max(1, min(workers, cpus, len(todo), int(mem_gb() / WORKER_GB))) | |
| threads = THREADS or max(1, cpus // workers) | |
| log(f"cpus={cpus} -> workers={workers} x threads={threads}") | |
| order = todo[np.argsort(lens[todo], kind="stable")] # global row ids, ascending length | |
| sorted_seqs = [seqs[i] for i in order] | |
| sorted_lens = lens[order] | |
| results = [] | |
| import multiprocessing as mp | |
| ctx = mp.get_context("spawn") | |
| if workers > 1: | |
| procs = [] | |
| try: | |
| cursor = ctx.Value("i", 0) | |
| rates = ctx.Array("d", workers) | |
| barrier = ctx.Barrier(workers) | |
| out_q = ctx.Queue() | |
| procs = [ctx.Process(target=worker, args=(w, sorted_seqs, sorted_lens, T0, threads, | |
| cursor, rates, barrier, out_q)) | |
| for w in range(workers)] | |
| for p_ in procs: | |
| p_.start() | |
| pending, hard_stop = workers, T0 + TIME_BUDGET - 1 | |
| while pending and time.time() < hard_stop: | |
| try: | |
| results.append(out_q.get(timeout=2)) | |
| pending -= 1 | |
| except queue.Empty: | |
| if not any(p_.is_alive() for p_ in procs): | |
| while True: | |
| try: | |
| results.append(out_q.get(timeout=1)) | |
| except queue.Empty: | |
| break | |
| break | |
| for p_ in procs: | |
| if p_.is_alive(): | |
| p_.terminate() | |
| except Exception as e: | |
| log(f"multiprocessing failed ({e!r}); falling back to a single process") | |
| for p_ in procs: | |
| if p_.is_alive(): | |
| p_.terminate() | |
| results = [] | |
| if not results: | |
| workers = 1 | |
| if workers == 1: | |
| q1 = queue.Queue() | |
| worker(0, sorted_seqs, sorted_lens, T0, cpus, ctx.Value("i", 0), ctx.Array("d", 1), | |
| ctx.Barrier(1), q1) | |
| results = [q1.get()] | |
| got = swapped = 0 | |
| for _, part in results: | |
| for s, (best, conf) in part: | |
| i = order[s] | |
| if lk_ec[i] is not None and lk_sc[i] >= LK_MID and conf < CONF: | |
| preds[i] = [lk_ec[i]] | |
| swapped += 1 | |
| else: | |
| preds[i] = best | |
| got += 1 | |
| log(f"transformer predicted {got}/{len(todo)} rows ({len(todo) - got} use fallback EC); " | |
| f"{swapped} low-confidence rows replaced by lookup hit") | |
| Path(output_path).parent.mkdir(parents=True, exist_ok=True) | |
| with Path(output_path).open("w") as f: | |
| for key, p in zip(frame.id, preds): | |
| f.write(json.dumps({"id": key, "prediction": p}) + "\n") | |
| log("done") | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--input", required=True, help="Input Parquet file") | |
| parser.add_argument("--output", required=True, help="Output JSONL file") | |
| args = parser.parse_args() | |
| predict_file(args.input, args.output) |