"""Build an evo2_flash engine and generate. Two lifetimes: - ZeroGPU (the Space as published): the GPU is attached only inside an @spaces.GPU call and the engine cannot outlive it, so each call builds an engine (~11 s once the worker has compiled its kernels, ~2-4 min on a container's first call) and releases it before returning. Batch work into one call to pay that once. - Dedicated GPU (the Space duplicated onto L4/L40S/A100 hardware, or run locally): one engine is built on first use and kept resident, so calls pay only generation. """ from __future__ import annotations import gc import os import threading import time # FlashInfer's sampler JIT-compiles with whatever nvcc is on PATH; vLLM's PyTorch # sampler needs no compiler. os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0") # On sm_100+ (ZeroGPU's RTX Pro 6000 is sm_120) vLLM prefers FlashInfer's FP8 GEMM, # whose prebuilt sm_120 library links libcudart.so.13 and cannot load on this # CUDA 12.9 stack. CUTLASS (what Hopper uses) is next in line. os.environ.setdefault("VLLM_DISABLED_KERNELS", "FlashInferFP8ScaledMMLinearKernel") MODEL = os.environ.get("EVO2_MODEL", "evo2_7b") ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() in ("1", "true") RESIDENT = os.environ.get("EVO2_RESIDENT", "0" if ZERO_GPU else "1") == "1" # A resident engine is sized once for the largest request it will take. # 163,840 covers the mito example (~135 kb prompt + 16 kb of generation). RESIDENT_MAX_LEN = int(os.environ.get("EVO2_RESIDENT_MAX_LEN", "163840")) _resident = None _lock = threading.Lock() def download_checkpoint() -> str: """Fetch the published checkpoint (no GPU needed); returns its directory.""" from evo2_flash.api import resolve_checkpoint return resolve_checkpoint(MODEL) def _build(ckpt: str, max_len: int): from evo2_flash import Evo2 eager = os.environ.get("EVO2_ENFORCE_EAGER", "0") == "1" kw = {"enforce_eager": True} if eager else {} return Evo2(ckpt, max_model_len=max_len, **kw) def _teardown(model) -> None: try: model.llm.llm_engine.engine_core.shutdown() except Exception: pass del model gc.collect() try: from vllm.distributed.parallel_state import (destroy_distributed_environment, destroy_model_parallel) destroy_model_parallel() destroy_distributed_environment() except Exception: pass import torch gc.collect() if torch.cuda.is_available(): torch.cuda.empty_cache() def run(ckpt: str, prompts: list[str], num_sequences: int, n_tokens: int, *, temperature: float, top_k: int, top_p: float, seed: int | None, log=print) -> tuple[list[str], list[float], list[int], dict]: """Generate `num_sequences` continuations of every prompt. Returns (sequences, mean logprobs, prompt index per design, timings). """ global _resident import torch t = {"engine_s": 0.0} need = max(len(p) for p in prompts) + n_tokens + 64 with _lock: s = time.perf_counter() if RESIDENT: if need > RESIDENT_MAX_LEN: raise ValueError(f"request needs {need:,} positions; this engine holds " f"{RESIDENT_MAX_LEN:,} (EVO2_RESIDENT_MAX_LEN)") if _resident is None: _resident = _build(ckpt, RESIDENT_MAX_LEN) model = _resident else: model = _build(ckpt, max(8192, -(-need // 8192) * 8192)) # next multiple of 8192 t["engine_s"] = time.perf_counter() - s t["resident"] = RESIDENT log(f"engine ready in {t['engine_s']:.1f} s on {torch.cuda.get_device_name()} " f"({'resident' if RESIDENT else 'per call'})") try: batch = [p for p in prompts for _ in range(num_sequences)] which = [i for i in range(len(prompts)) for _ in range(num_sequences)] s = time.perf_counter() out = model.generate(batch, n_tokens=n_tokens, temperature=temperature, top_k=top_k, top_p=top_p, seed=seed, verbose=0) t["generate_s"] = time.perf_counter() - s return list(out.sequences), [float(x) for x in out.logprobs_mean], which, t finally: if not RESIDENT: _teardown(model)