evo2-icl / engine.py
gbrixi's picture
Examples: replace MS2/M13 with primate mitogenomes; size per-call engines to the request
bc3622d verified
Raw History Blame Contribute Delete
4.44 kB
"""Build an evo2_flash engine and generate.
Two lifetimes:
- ZeroGPU (the Space as published): the GPU is attached only inside an
@spaces.GPU call and the engine cannot outlive it, so each call builds an engine
(~11 s once the worker has compiled its kernels, ~2-4 min on a container's first
call) and releases it before returning. Batch work into one call to pay that once.
- Dedicated GPU (the Space duplicated onto L4/L40S/A100 hardware, or run locally):
one engine is built on first use and kept resident, so calls pay only generation.
"""
from __future__ import annotations
import gc
import os
import threading
import time
# FlashInfer's sampler JIT-compiles with whatever nvcc is on PATH; vLLM's PyTorch
# sampler needs no compiler.
os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
# On sm_100+ (ZeroGPU's RTX Pro 6000 is sm_120) vLLM prefers FlashInfer's FP8 GEMM,
# whose prebuilt sm_120 library links libcudart.so.13 and cannot load on this
# CUDA 12.9 stack. CUTLASS (what Hopper uses) is next in line.
os.environ.setdefault("VLLM_DISABLED_KERNELS", "FlashInferFP8ScaledMMLinearKernel")
MODEL = os.environ.get("EVO2_MODEL", "evo2_7b")
ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() in ("1", "true")
RESIDENT = os.environ.get("EVO2_RESIDENT", "0" if ZERO_GPU else "1") == "1"
# A resident engine is sized once for the largest request it will take.
# 163,840 covers the mito example (~135 kb prompt + 16 kb of generation).
RESIDENT_MAX_LEN = int(os.environ.get("EVO2_RESIDENT_MAX_LEN", "163840"))
_resident = None
_lock = threading.Lock()
def download_checkpoint() -> str:
"""Fetch the published checkpoint (no GPU needed); returns its directory."""
from evo2_flash.api import resolve_checkpoint
return resolve_checkpoint(MODEL)
def _build(ckpt: str, max_len: int):
from evo2_flash import Evo2
eager = os.environ.get("EVO2_ENFORCE_EAGER", "0") == "1"
kw = {"enforce_eager": True} if eager else {}
return Evo2(ckpt, max_model_len=max_len, **kw)
def _teardown(model) -> None:
try:
model.llm.llm_engine.engine_core.shutdown()
except Exception:
pass
del model
gc.collect()
try:
from vllm.distributed.parallel_state import (destroy_distributed_environment,
destroy_model_parallel)
destroy_model_parallel()
destroy_distributed_environment()
except Exception:
pass
import torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
def run(ckpt: str, prompts: list[str], num_sequences: int, n_tokens: int, *,
temperature: float, top_k: int, top_p: float, seed: int | None,
log=print) -> tuple[list[str], list[float], list[int], dict]:
"""Generate `num_sequences` continuations of every prompt.
Returns (sequences, mean logprobs, prompt index per design, timings).
"""
global _resident
import torch
t = {"engine_s": 0.0}
need = max(len(p) for p in prompts) + n_tokens + 64
with _lock:
s = time.perf_counter()
if RESIDENT:
if need > RESIDENT_MAX_LEN:
raise ValueError(f"request needs {need:,} positions; this engine holds "
f"{RESIDENT_MAX_LEN:,} (EVO2_RESIDENT_MAX_LEN)")
if _resident is None:
_resident = _build(ckpt, RESIDENT_MAX_LEN)
model = _resident
else:
model = _build(ckpt, max(8192, -(-need // 8192) * 8192)) # next multiple of 8192
t["engine_s"] = time.perf_counter() - s
t["resident"] = RESIDENT
log(f"engine ready in {t['engine_s']:.1f} s on {torch.cuda.get_device_name()} "
f"({'resident' if RESIDENT else 'per call'})")
try:
batch = [p for p in prompts for _ in range(num_sequences)]
which = [i for i in range(len(prompts)) for _ in range(num_sequences)]
s = time.perf_counter()
out = model.generate(batch, n_tokens=n_tokens, temperature=temperature,
top_k=top_k, top_p=top_p, seed=seed, verbose=0)
t["generate_s"] = time.perf_counter() - s
return list(out.sequences), [float(x) for x in out.logprobs_mean], which, t
finally:
if not RESIDENT:
_teardown(model)