from __future__ import annotations import asyncio import logging import threading from dataclasses import dataclass from huggingface_hub import hf_hub_download from llama_cpp import Llama from app.config import ( HF_TOKEN, LLAMA_CONTEXT_SIZE, LLAMA_GPU_LAYERS, LLAMA_THREADS, MAX_NEW_TOKENS, MODEL_FILENAME, MODEL_REPO_ID, REPEAT_PENALTY, SEED, TEMPERATURE, TOP_P, ) logger = logging.getLogger(__name__) class ModelNotReadyError(RuntimeError): pass @dataclass(frozen=True) class Completion: text: str input_tokens: int output_tokens: int class ModelRuntime: """Owns the single in-process GGUF model instance.""" def __init__(self) -> None: self.llm: Llama | None = None self.model_path: str | None = None self.load_error: str | None = None # A thread lock (not asyncio.Lock) because generation runs in a # worker thread; a Llama instance is not safe for concurrent use. self.inference_lock = threading.Lock() def download_model(self) -> str: path = hf_hub_download( repo_id=MODEL_REPO_ID, filename=MODEL_FILENAME, repo_type="model", token=HF_TOKEN or None, ) self.model_path = path return path def load(self) -> None: model_path = self.download_model() self.llm = Llama( model_path=model_path, n_ctx=LLAMA_CONTEXT_SIZE, n_threads=LLAMA_THREADS, n_gpu_layers=LLAMA_GPU_LAYERS, verbose=False, ) async def start(self) -> None: """ Download and load the model once. A failure is recorded rather than raised so the API still starts and /health can report the model as not ready. """ try: await asyncio.to_thread(self.load) self.load_error = None except Exception as exc: self.llm = None self.load_error = f"{type(exc).__name__}: {exc}" logger.exception("Failed to load the Progression model.") async def stop(self) -> None: llm, self.llm = self.llm, None if llm is not None: with self.inference_lock: llm.close() def ready(self) -> bool: return self.llm is not None def _tokenize(self, llm: Llama, text: str, *, prompt: bool) -> list[int]: # Prompt tokenization mirrors what Llama.create_completion does # internally (BOS handling + special-token parsing), so the count # equals what the model actually evaluates. return llm.tokenize( text.encode("utf-8"), add_bos=prompt, special=prompt, ) def _complete(self, prompt: str) -> Completion: with self.inference_lock: llm = self.llm if llm is None: raise ModelNotReadyError("Progression model is not loaded.") input_tokens = len(self._tokenize(llm, prompt, prompt=True)) result = llm.create_completion( prompt, max_tokens=MAX_NEW_TOKENS, temperature=TEMPERATURE, top_p=TOP_P, repeat_penalty=REPEAT_PENALTY, seed=SEED, echo=False, ) text = result["choices"][0]["text"] output_tokens = len(self._tokenize(llm, text, prompt=False)) return Completion( text=text.strip(), input_tokens=input_tokens, output_tokens=output_tokens, ) async def complete(self, prompt: str) -> Completion: """ Raw completion (no chat template) with the internal generation settings. The blocking llama.cpp call runs in a worker thread so the event loop stays responsive; the lock serializes access to the model. """ return await asyncio.to_thread(self._complete, prompt) runtime = ModelRuntime()