Spaces:
Sleeping
Sleeping
File size: 4,025 Bytes
63d4e3a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | from __future__ import annotations
import asyncio
import logging
import threading
from dataclasses import dataclass
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
from app.config import (
HF_TOKEN,
LLAMA_CONTEXT_SIZE,
LLAMA_GPU_LAYERS,
LLAMA_THREADS,
MAX_NEW_TOKENS,
MODEL_FILENAME,
MODEL_REPO_ID,
REPEAT_PENALTY,
SEED,
TEMPERATURE,
TOP_P,
)
logger = logging.getLogger(__name__)
class ModelNotReadyError(RuntimeError):
pass
@dataclass(frozen=True)
class Completion:
text: str
input_tokens: int
output_tokens: int
class ModelRuntime:
"""Owns the single in-process GGUF model instance."""
def __init__(self) -> None:
self.llm: Llama | None = None
self.model_path: str | None = None
self.load_error: str | None = None
# A thread lock (not asyncio.Lock) because generation runs in a
# worker thread; a Llama instance is not safe for concurrent use.
self.inference_lock = threading.Lock()
def download_model(self) -> str:
path = hf_hub_download(
repo_id=MODEL_REPO_ID,
filename=MODEL_FILENAME,
repo_type="model",
token=HF_TOKEN or None,
)
self.model_path = path
return path
def load(self) -> None:
model_path = self.download_model()
self.llm = Llama(
model_path=model_path,
n_ctx=LLAMA_CONTEXT_SIZE,
n_threads=LLAMA_THREADS,
n_gpu_layers=LLAMA_GPU_LAYERS,
verbose=False,
)
async def start(self) -> None:
"""
Download and load the model once.
A failure is recorded rather than raised so the API still starts
and /health can report the model as not ready.
"""
try:
await asyncio.to_thread(self.load)
self.load_error = None
except Exception as exc:
self.llm = None
self.load_error = f"{type(exc).__name__}: {exc}"
logger.exception("Failed to load the Progression model.")
async def stop(self) -> None:
llm, self.llm = self.llm, None
if llm is not None:
with self.inference_lock:
llm.close()
def ready(self) -> bool:
return self.llm is not None
def _tokenize(self, llm: Llama, text: str, *, prompt: bool) -> list[int]:
# Prompt tokenization mirrors what Llama.create_completion does
# internally (BOS handling + special-token parsing), so the count
# equals what the model actually evaluates.
return llm.tokenize(
text.encode("utf-8"),
add_bos=prompt,
special=prompt,
)
def _complete(self, prompt: str) -> Completion:
with self.inference_lock:
llm = self.llm
if llm is None:
raise ModelNotReadyError("Progression model is not loaded.")
input_tokens = len(self._tokenize(llm, prompt, prompt=True))
result = llm.create_completion(
prompt,
max_tokens=MAX_NEW_TOKENS,
temperature=TEMPERATURE,
top_p=TOP_P,
repeat_penalty=REPEAT_PENALTY,
seed=SEED,
echo=False,
)
text = result["choices"][0]["text"]
output_tokens = len(self._tokenize(llm, text, prompt=False))
return Completion(
text=text.strip(),
input_tokens=input_tokens,
output_tokens=output_tokens,
)
async def complete(self, prompt: str) -> Completion:
"""
Raw completion (no chat template) with the internal generation
settings.
The blocking llama.cpp call runs in a worker thread so the event
loop stays responsive; the lock serializes access to the model.
"""
return await asyncio.to_thread(self._complete, prompt)
runtime = ModelRuntime()
|