Download engine.py from gbrixi/evo2-icl: direct link, hf CLI and curl.
- Browser
- Download file 4.44 kB
-
https://huggingface.co/spaces/gbrixi/evo2-icl/resolve/main/engine.py
- Command line
-
hf download hf://spaces/gbrixi/evo2-icl/engine.py
-
curl -L -o engine.py https://huggingface.co/spaces/gbrixi/evo2-icl/resolve/main/engine.py
4.44 kB
| """Build an evo2_flash engine and generate. | |
| Two lifetimes: | |
| - ZeroGPU (the Space as published): the GPU is attached only inside an | |
| @spaces.GPU call and the engine cannot outlive it, so each call builds an engine | |
| (~11 s once the worker has compiled its kernels, ~2-4 min on a container's first | |
| call) and releases it before returning. Batch work into one call to pay that once. | |
| - Dedicated GPU (the Space duplicated onto L4/L40S/A100 hardware, or run locally): | |
| one engine is built on first use and kept resident, so calls pay only generation. | |
| """ | |
| from __future__ import annotations | |
| import gc | |
| import os | |
| import threading | |
| import time | |
| # FlashInfer's sampler JIT-compiles with whatever nvcc is on PATH; vLLM's PyTorch | |
| # sampler needs no compiler. | |
| os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") | |
| os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0") | |
| # On sm_100+ (ZeroGPU's RTX Pro 6000 is sm_120) vLLM prefers FlashInfer's FP8 GEMM, | |
| # whose prebuilt sm_120 library links libcudart.so.13 and cannot load on this | |
| # CUDA 12.9 stack. CUTLASS (what Hopper uses) is next in line. | |
| os.environ.setdefault("VLLM_DISABLED_KERNELS", "FlashInferFP8ScaledMMLinearKernel") | |
| MODEL = os.environ.get("EVO2_MODEL", "evo2_7b") | |
| ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() in ("1", "true") | |
| RESIDENT = os.environ.get("EVO2_RESIDENT", "0" if ZERO_GPU else "1") == "1" | |
| # A resident engine is sized once for the largest request it will take. | |
| # 163,840 covers the mito example (~135 kb prompt + 16 kb of generation). | |
| RESIDENT_MAX_LEN = int(os.environ.get("EVO2_RESIDENT_MAX_LEN", "163840")) | |
| _resident = None | |
| _lock = threading.Lock() | |
| def download_checkpoint() -> str: | |
| """Fetch the published checkpoint (no GPU needed); returns its directory.""" | |
| from evo2_flash.api import resolve_checkpoint | |
| return resolve_checkpoint(MODEL) | |
| def _build(ckpt: str, max_len: int): | |
| from evo2_flash import Evo2 | |
| eager = os.environ.get("EVO2_ENFORCE_EAGER", "0") == "1" | |
| kw = {"enforce_eager": True} if eager else {} | |
| return Evo2(ckpt, max_model_len=max_len, **kw) | |
| def _teardown(model) -> None: | |
| try: | |
| model.llm.llm_engine.engine_core.shutdown() | |
| except Exception: | |
| pass | |
| del model | |
| gc.collect() | |
| try: | |
| from vllm.distributed.parallel_state import (destroy_distributed_environment, | |
| destroy_model_parallel) | |
| destroy_model_parallel() | |
| destroy_distributed_environment() | |
| except Exception: | |
| pass | |
| import torch | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| def run(ckpt: str, prompts: list[str], num_sequences: int, n_tokens: int, *, | |
| temperature: float, top_k: int, top_p: float, seed: int | None, | |
| log=print) -> tuple[list[str], list[float], list[int], dict]: | |
| """Generate `num_sequences` continuations of every prompt. | |
| Returns (sequences, mean logprobs, prompt index per design, timings). | |
| """ | |
| global _resident | |
| import torch | |
| t = {"engine_s": 0.0} | |
| need = max(len(p) for p in prompts) + n_tokens + 64 | |
| with _lock: | |
| s = time.perf_counter() | |
| if RESIDENT: | |
| if need > RESIDENT_MAX_LEN: | |
| raise ValueError(f"request needs {need:,} positions; this engine holds " | |
| f"{RESIDENT_MAX_LEN:,} (EVO2_RESIDENT_MAX_LEN)") | |
| if _resident is None: | |
| _resident = _build(ckpt, RESIDENT_MAX_LEN) | |
| model = _resident | |
| else: | |
| model = _build(ckpt, max(8192, -(-need // 8192) * 8192)) # next multiple of 8192 | |
| t["engine_s"] = time.perf_counter() - s | |
| t["resident"] = RESIDENT | |
| log(f"engine ready in {t['engine_s']:.1f} s on {torch.cuda.get_device_name()} " | |
| f"({'resident' if RESIDENT else 'per call'})") | |
| try: | |
| batch = [p for p in prompts for _ in range(num_sequences)] | |
| which = [i for i in range(len(prompts)) for _ in range(num_sequences)] | |
| s = time.perf_counter() | |
| out = model.generate(batch, n_tokens=n_tokens, temperature=temperature, | |
| top_k=top_k, top_p=top_p, seed=seed, verbose=0) | |
| t["generate_s"] = time.perf_counter() - s | |
| return list(out.sequences), [float(x) for x in out.logprobs_mean], which, t | |
| finally: | |
| if not RESIDENT: | |
| _teardown(model) | |