activity-progression-api / app /model_runtime.py
mjpsm's picture
Deploy Progression API (llama-cpp-python)
63d4e3a verified
Raw History Blame Contribute Delete
4.03 kB
from __future__ import annotations
import asyncio
import logging
import threading
from dataclasses import dataclass
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
from app.config import (
HF_TOKEN,
LLAMA_CONTEXT_SIZE,
LLAMA_GPU_LAYERS,
LLAMA_THREADS,
MAX_NEW_TOKENS,
MODEL_FILENAME,
MODEL_REPO_ID,
REPEAT_PENALTY,
SEED,
TEMPERATURE,
TOP_P,
)
logger = logging.getLogger(__name__)
class ModelNotReadyError(RuntimeError):
pass
@dataclass(frozen=True)
class Completion:
text: str
input_tokens: int
output_tokens: int
class ModelRuntime:
"""Owns the single in-process GGUF model instance."""
def __init__(self) -> None:
self.llm: Llama | None = None
self.model_path: str | None = None
self.load_error: str | None = None
# A thread lock (not asyncio.Lock) because generation runs in a
# worker thread; a Llama instance is not safe for concurrent use.
self.inference_lock = threading.Lock()
def download_model(self) -> str:
path = hf_hub_download(
repo_id=MODEL_REPO_ID,
filename=MODEL_FILENAME,
repo_type="model",
token=HF_TOKEN or None,
)
self.model_path = path
return path
def load(self) -> None:
model_path = self.download_model()
self.llm = Llama(
model_path=model_path,
n_ctx=LLAMA_CONTEXT_SIZE,
n_threads=LLAMA_THREADS,
n_gpu_layers=LLAMA_GPU_LAYERS,
verbose=False,
)
async def start(self) -> None:
"""
Download and load the model once.
A failure is recorded rather than raised so the API still starts
and /health can report the model as not ready.
"""
try:
await asyncio.to_thread(self.load)
self.load_error = None
except Exception as exc:
self.llm = None
self.load_error = f"{type(exc).__name__}: {exc}"
logger.exception("Failed to load the Progression model.")
async def stop(self) -> None:
llm, self.llm = self.llm, None
if llm is not None:
with self.inference_lock:
llm.close()
def ready(self) -> bool:
return self.llm is not None
def _tokenize(self, llm: Llama, text: str, *, prompt: bool) -> list[int]:
# Prompt tokenization mirrors what Llama.create_completion does
# internally (BOS handling + special-token parsing), so the count
# equals what the model actually evaluates.
return llm.tokenize(
text.encode("utf-8"),
add_bos=prompt,
special=prompt,
)
def _complete(self, prompt: str) -> Completion:
with self.inference_lock:
llm = self.llm
if llm is None:
raise ModelNotReadyError("Progression model is not loaded.")
input_tokens = len(self._tokenize(llm, prompt, prompt=True))
result = llm.create_completion(
prompt,
max_tokens=MAX_NEW_TOKENS,
temperature=TEMPERATURE,
top_p=TOP_P,
repeat_penalty=REPEAT_PENALTY,
seed=SEED,
echo=False,
)
text = result["choices"][0]["text"]
output_tokens = len(self._tokenize(llm, text, prompt=False))
return Completion(
text=text.strip(),
input_tokens=input_tokens,
output_tokens=output_tokens,
)
async def complete(self, prompt: str) -> Completion:
"""
Raw completion (no chat template) with the internal generation
settings.
The blocking llama.cpp call runs in a worker thread so the event
loop stays responsive; the lock serializes access to the model.
"""
return await asyncio.to_thread(self._complete, prompt)
runtime = ModelRuntime()