import argparse import json import math import os import queue import time import traceback from pathlib import Path import numpy as np import pandas as pd import torch from transformers import AutoTokenizer from model import load_model, restrict_classes, build_variants, fuse_qkv_verified from inputs import prepare_inputs T0 = time.time() ROOT = Path(__file__).parent WEIGHTS = ROOT / "weights" TIME_BUDGET = float(os.environ.get("EC_TIME_BUDGET", 660)) BATCH = int(os.environ.get("EC_BATCH", 64)) # max rows per batch TOK_BUDGET = int(os.environ.get("EC_TOK_BUDGET", 2048)) # max tokens per batch (cache friendly) THRESH = float(os.environ.get("EC_THRESH", 0.5)) # softmax + 0.5 => top-1 only (what training optimised) MAX_EXTRA = 2 PRECISION = os.environ.get("EC_PRECISION", "auto") ACC_TOL = float(os.environ.get("EC_ACC_TOL", 1.12)) QUANT_FFN = int(os.environ.get("EC_QUANT_FFN_ONLY", 0)) SDPA = int(os.environ.get("EC_SDPA", 1)) FUSE_QKV = int(os.environ.get("EC_FUSE_QKV", 1)) USE_ORT = int(os.environ.get("EC_ORT", 1)) # use weights/onnx/*.onnx when present ONNX_DIR = WEIGHTS / "onnx" MIN_LEN = int(os.environ.get("EC_MIN_LEN", 64)) SAFETY = float(os.environ.get("EC_SAFETY", 8)) PLAN_MARGIN = float(os.environ.get("EC_PLAN_MARGIN", 0.96)) WORKERS = int(os.environ.get("EC_WORKERS", 0)) THREADS = int(os.environ.get("EC_THREADS", 0)) WORKER_GB = float(os.environ.get("EC_WORKER_GB", 1.2)) BENCH_N = int(os.environ.get("EC_BENCH_N", 8)) BENCH_LEN = int(os.environ.get("EC_BENCH_LEN", 160)) L0 = float(os.environ.get("EC_L0", 800)) # cost(len) ~ len * (1 + len / L0) MAX_CAP = int(os.environ.get("EC_MAX_CAP", 0)) # experiments only: force a length cap (0 = off) LOOKUP = int(os.environ.get("EC_LOOKUP", 1)) LK_SKIP = float(os.environ.get("EC_LOOKUP_SKIP", 0.6)) # score >= this: take the training-set hit, skip the transformer LK_MID = float(os.environ.get("EC_LOOKUP_MID", 0.15)) # score >= this AND model confidence < EC_CONF: take the hit CONF = float(os.environ.get("EC_CONF", 0.6)) def log(msg: str, tag: str = ""): print(f"[{time.time() - T0:6.1f}s]{tag} {msg}", flush=True) def n_cpus() -> int: try: n = len(os.sched_getaffinity(0)) except AttributeError: n = os.cpu_count() or 2 quota = None try: q, p = Path("/sys/fs/cgroup/cpu.max").read_text().split() if q != "max": quota = int(q) / int(p) except Exception: try: q = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read_text()) p = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read_text()) if q > 0: quota = q / p except Exception: pass log(f"cpu affinity={n}, cgroup quota={quota}") if quota: n = min(n, max(1, math.ceil(quota))) return n def mem_gb() -> float: vals = [] try: for line in Path("/proc/meminfo").read_text().splitlines(): if line.startswith("MemAvailable"): vals.append(int(line.split()[1]) / 1e6) except Exception: pass for p in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"): try: v = Path(p).read_text().strip() if v.isdigit(): vals.append(int(v) / 1e9) except Exception: pass return min(vals) if vals else 1e9 def cpu_flags() -> set: try: for line in Path("/proc/cpuinfo").read_text().splitlines(): if line.startswith("flags"): return set(line.split(":", 1)[1].split()) except Exception: pass return set() def candidate_precisions(): """Ordered most-accurate first; the fastest kind wins unless an earlier one is within ACC_TOL of it.""" if PRECISION != "auto": return [PRECISION] ort_ok = False if USE_ORT and ONNX_DIR.exists(): try: import onnxruntime # noqa: F401 ort_ok = True except Exception: pass kinds = ["fp32"] if ort_ok and (ONNX_DIR / "esm_fp32.onnx").exists(): kinds.append("ort_fp32") if cpu_flags() & {"amx_bf16", "avx512_bf16"}: kinds.append("bf16") kinds.append("int8") if ort_ok and (ONNX_DIR / "esm_int8.onnx").exists(): kinds.append("ort_int8") return kinds def cost_units(lens, cap): e = np.minimum(lens, cap) return float((e * (1.0 + e / L0)).sum()) def choose_cap(rem_lens, norm_rate, time_left, max_len): """Largest cap such that the remaining rows fit in the time left (length-aware cost model).""" budget = max(0.0, time_left) * PLAN_MARGIN * norm_rate lo, hi = MIN_LEN, max_len if cost_units(rem_lens, hi) <= budget: return hi while lo < hi: mid = (lo + hi + 1) // 2 if cost_units(rem_lens, mid) <= budget: lo = mid else: hi = mid - 1 return lo def worker(wid, seqs, lens, t0_abs, threads, cursor, rates_arr, barrier, out_q): """All workers pull batches from ONE shared length-sorted queue (perfect load balance, one global cap). Batches are built from rows of IDENTICAL (capped) length whenever possible: zero padding, no attention mask needed -> fastest attention path, and no wasted compute. """ global T0 T0 = t0_abs tag = f" [w{wid}]" done = {} try: torch.set_num_threads(threads) try: torch.set_num_interop_threads(1) except RuntimeError: pass torch.set_flush_denormal(True) tok = AutoTokenizer.from_pretrained(str(WEIGHTS / "esm2")) n = len(seqs) k = 1 + MAX_EXTRA deadline = t0_abs + TIME_BUDGET - SAFETY vocab = set(json.loads((ROOT / "ec_vocabulary.json").read_text())) def run(idx, length, fwd, l4): t = time.time() enc = tok([seqs[i] for i in idx], padding="longest", truncation=True, max_length=length, return_tensors="pt") mask = enc["attention_mask"] if bool(mask.all()): # equal lengths: no padding, no mask mask = None with torch.inference_mode(): logits = fwd(enc["input_ids"], mask).float() probs = torch.softmax(logits, dim=-1) top_p, top_j = probs.topk(min(k, probs.shape[1]), dim=1) top_p, top_j = top_p.tolist(), top_j.tolist() for row, i in enumerate(idx): best = [l4[top_j[row][0]]] if THRESH <= 1.0: for p, j in zip(top_p[row][1:], top_j[row][1:]): if p >= THRESH: best.append(l4[j]) done[i] = (best, top_p[row][0]) return enc["input_ids"].numel(), time.time() - t start = min(int(0.6 * n), max(0, n - BENCH_N)) probe = list(range(start, min(n, start + BENCH_N))) kinds = candidate_precisions() # ---- build + benchmark; if the fused-QKV int8 path misbehaves, redo without fusion for fuse in ((True, False) if FUSE_QKV else (False,)): model, meta, classes = load_model(WEIGHTS, sdpa=bool(SDPA)) max_len = meta["max_len"] l4 = classes["l4"] keep = [i for i, ec in enumerate(l4) if ec in vocab] if not keep: raise RuntimeError("No predicted class is in ec_vocabulary.json") l4 = [l4[i] for i in keep] restrict_classes(model, keep) n_fused = 0 if fuse: enc = tok([seqs[i] for i in probe[:3]], padding="longest", truncation=True, max_length=96, return_tensors="pt") model, n_fused = fuse_qkv_verified(model, enc["input_ids"], enc["attention_mask"]) variants = build_variants(model, kinds, bool(QUANT_FFN), ONNX_DIR, threads) rates, ref = {}, None def logits_of(idx, length, f): enc_ = tok([seqs[i] for i in idx], padding="longest", truncation=True, max_length=length, return_tensors="pt") with torch.inference_mode(): return f(enc_["input_ids"], enc_["attention_mask"]).float() for kind in kinds: try: if kind.startswith("ort") and "fp32" in variants: # parity check vs torch fp32 pidx = probe + [0] # row 0 is the shortest -> the batch is padded if ref is None: ref = logits_of(pidx, 128, variants["fp32"]) out = logits_of(pidx, 128, variants[kind]) rel = ((out - ref).norm() / ref.norm()).item() if not rel < (0.02 if kind == "ort_fp32" else 0.35): log(f"{kind} rejected: relative logit error {rel:.3f}", tag) continue run(probe[:2], 64, variants[kind], l4) toks, secs = run(probe, min(BENCH_LEN, max_len), variants[kind], l4) rates[kind] = toks / secs except Exception as e: log(f"{kind} failed during benchmark (fused={n_fused}): {e!r}", tag) if rates and ("int8" in rates or "int8" not in kinds or not n_fused): break done.clear() best = max(rates.values()) choice = next(kind for kind in kinds if kind in rates and rates[kind] >= best / ACC_TOL) fwd = variants[choice] variants = model = None rates_arr[wid] = rates[choice] * 0.85 * (1 + min(BENCH_LEN, max_len) / L0) if wid == 0: log(f"rates tok/s { {k_: round(v) for k_, v in rates.items()} } -> {choice}, " f"threads={threads}, fused_qkv_layers={n_fused}", tag) try: barrier.wait(timeout=90) except Exception: pass def claim(): with cursor.get_lock(): c = cursor.value now = time.time() if c >= n or now > deadline: return None cap = choose_cap(lens[c:], sum(rates_arr[:]), deadline - now, min(max_len, MAX_CAP) if MAX_CAP else max_len) L, e = min(lens[c], cap), c + 1 while e < n and e - c < BATCH and min(lens[e], cap) == L and L * (e + 1 - c) <= TOK_BUDGET: e += 1 if e - c < 3: # sparse length: allow a few near-equal lengths (uses the mask path) while e < n and e - c < BATCH and min(lens[e], cap) - L <= 8 \ and min(lens[e], cap) * (e + 1 - c) <= TOK_BUDGET: e += 1 cursor.value = e return c, e, cap capped_rows = nb = 0 cap, last_log = max_len, time.time() while True: job = claim() if job is None: break c, e, cap = job capped_rows += int((lens[c:e] > cap).sum()) toks, secs = run(list(range(c, e)), cap, fwd, l4) if toks >= 1024 and secs > 0: mean_len = toks / (e - c) rates_arr[wid] = 0.7 * rates_arr[wid] + 0.3 * (toks / secs) * (1 + mean_len / L0) nb += 1 if wid == 0 and time.time() - last_log > 45: last_log = time.time() log(f"row {c}/{n} cap={cap} agg norm rate={sum(rates_arr[:]):.0f}", tag) log(f"finished: {len(done)} rows, {nb} batches, last cap={cap}, truncated rows={capped_rows}", tag) except Exception: log("worker crashed:\n" + traceback.format_exc(), tag) finally: out_q.put((wid, list(done.items()))) def predict_file(input_path: str, output_path: str): frame = prepare_inputs(pd.read_parquet(input_path)) n = len(frame) log(f"{n} unique inputs") classes = json.loads((WEIGHTS / "classes.json").read_text()) fallback = classes["fallback_ec"] seqs = [str(s).upper() for s in frame.sequence] lens = np.array([len(s) + 2 for s in seqs]) if n: log(f"token length p50/p90/p99 = {np.percentile(lens, [50, 90, 99]).astype(int).tolist()}, " f"mean={lens.mean():.0f}") preds = [[fallback]] * n lk_ec, lk_sc = [None] * n, np.zeros(n) todo = np.arange(n) lk_dir = WEIGHTS / "lookup" if n and LOOKUP and (lk_dir / "meta.json").exists(): try: from lookup import Lookup lk = Lookup(lk_dir) for i, s in enumerate(seqs): lk_ec[i], lk_sc[i] = lk.query(s) stats = {t: f"{(lk_sc >= t).mean():.1%}" for t in (0.06, 0.15, 0.3, 0.6, 0.9)} log(f"lookup done: share of rows with score >= t: {stats}") skip = lk_sc >= LK_SKIP for i in np.flatnonzero(skip): preds[i] = [lk_ec[i]] todo = np.flatnonzero(~skip) log(f"{int(skip.sum())} rows resolved by lookup, {len(todo)} go to the transformer") except Exception: log("lookup failed, continuing without it:\n" + traceback.format_exc()) lk_ec, lk_sc, todo = [None] * n, np.zeros(n), np.arange(n) if len(todo): cpus = n_cpus() workers = WORKERS or cpus workers = max(1, min(workers, cpus, len(todo), int(mem_gb() / WORKER_GB))) threads = THREADS or max(1, cpus // workers) log(f"cpus={cpus} -> workers={workers} x threads={threads}") order = todo[np.argsort(lens[todo], kind="stable")] # global row ids, ascending length sorted_seqs = [seqs[i] for i in order] sorted_lens = lens[order] results = [] import multiprocessing as mp ctx = mp.get_context("spawn") if workers > 1: procs = [] try: cursor = ctx.Value("i", 0) rates = ctx.Array("d", workers) barrier = ctx.Barrier(workers) out_q = ctx.Queue() procs = [ctx.Process(target=worker, args=(w, sorted_seqs, sorted_lens, T0, threads, cursor, rates, barrier, out_q)) for w in range(workers)] for p_ in procs: p_.start() pending, hard_stop = workers, T0 + TIME_BUDGET - 1 while pending and time.time() < hard_stop: try: results.append(out_q.get(timeout=2)) pending -= 1 except queue.Empty: if not any(p_.is_alive() for p_ in procs): while True: try: results.append(out_q.get(timeout=1)) except queue.Empty: break break for p_ in procs: if p_.is_alive(): p_.terminate() except Exception as e: log(f"multiprocessing failed ({e!r}); falling back to a single process") for p_ in procs: if p_.is_alive(): p_.terminate() results = [] if not results: workers = 1 if workers == 1: q1 = queue.Queue() worker(0, sorted_seqs, sorted_lens, T0, cpus, ctx.Value("i", 0), ctx.Array("d", 1), ctx.Barrier(1), q1) results = [q1.get()] got = swapped = 0 for _, part in results: for s, (best, conf) in part: i = order[s] if lk_ec[i] is not None and lk_sc[i] >= LK_MID and conf < CONF: preds[i] = [lk_ec[i]] swapped += 1 else: preds[i] = best got += 1 log(f"transformer predicted {got}/{len(todo)} rows ({len(todo) - got} use fallback EC); " f"{swapped} low-confidence rows replaced by lookup hit") Path(output_path).parent.mkdir(parents=True, exist_ok=True) with Path(output_path).open("w") as f: for key, p in zip(frame.id, preds): f.write(json.dumps({"id": key, "prediction": p}) + "\n") log("done") if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", required=True, help="Input Parquet file") parser.add_argument("--output", required=True, help="Output JSONL file") args = parser.parse_args() predict_file(args.input, args.output)