File size: 1,738 Bytes
e5f78fc 3a23c77 e5f78fc 3a23c77 e5f78fc 3a23c77 e5f78fc 3a23c77 e5f78fc 3a23c77 e5f78fc 3a23c77 e5f78fc 3a23c77 e5f78fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | """
llama-cpp-python bootstrap — MUST import before gradio or anything else.
2026-07-27: simplified after the great wheel hunt.
- PyPI = source-only (compiles 20+ min — the old "runs and runs" bug).
- abetlen CUDA indexes = abandoned (newest cu121 wheel is v0.2.59). Do NOT attempt.
- abetlen CPU index = current prebuilt wheels. That is the only fast path.
GPU speed returns via the Level-2 llama-server route (official llama.cpp CUDA
binaries + MTP flags) — separate spec, not this file.
"""
from __future__ import annotations
import os
import subprocess
import sys
CPU_WHEEL_INDEX = "https://abetlen.github.io/llama-cpp-python/whl/cpu"
def _pip(args: list) -> None:
cmd = [sys.executable, "-m", "pip", *args]
print(f"[LLAMA] $ {' '.join(cmd)}", flush=True)
env = os.environ.copy()
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
env["PIP_ROOT_USER_ACTION"] = "ignore"
subprocess.check_call(cmd, env=env)
def try_import_llama():
import llama_cpp # noqa: F401
return llama_cpp
def bootstrap_llama_cpp():
try:
mod = try_import_llama()
print("[LLAMA] Ready (prebuilt wheel already installed)", flush=True)
except ImportError:
print("[LLAMA] Not installed — installing CPU wheel (fast path)...", flush=True)
_pip([
"install", "--no-cache-dir", "llama-cpp-python==0.3.34",
"--extra-index-url", CPU_WHEEL_INDEX,
])
mod = try_import_llama()
# CPU wheel => CPU inference. Be explicit so nothing pretends otherwise.
os.environ.setdefault("DAVIDAU_N_GPU_LAYERS", "0")
print("[LLAMA] CPU mode (n_gpu_layers=0) — GPU speed is the Level-2 llama-server job", flush=True)
return mod
bootstrap_llama_cpp() |