Spaces:
Sleeping
Sleeping
Download app/model_runtime.py from mjpsm/activity-progression-api: direct link, hf CLI and curl.
- Browser
- Download file 4.03 kB
-
https://huggingface.co/spaces/mjpsm/activity-progression-api/resolve/main/app/model_runtime.py
- Command line
-
hf download hf://spaces/mjpsm/activity-progression-api/app/model_runtime.py
-
curl -L -o model_runtime.py https://huggingface.co/spaces/mjpsm/activity-progression-api/resolve/main/app/model_runtime.py
4.03 kB
| from __future__ import annotations | |
| import asyncio | |
| import logging | |
| import threading | |
| from dataclasses import dataclass | |
| from huggingface_hub import hf_hub_download | |
| from llama_cpp import Llama | |
| from app.config import ( | |
| HF_TOKEN, | |
| LLAMA_CONTEXT_SIZE, | |
| LLAMA_GPU_LAYERS, | |
| LLAMA_THREADS, | |
| MAX_NEW_TOKENS, | |
| MODEL_FILENAME, | |
| MODEL_REPO_ID, | |
| REPEAT_PENALTY, | |
| SEED, | |
| TEMPERATURE, | |
| TOP_P, | |
| ) | |
| logger = logging.getLogger(__name__) | |
| class ModelNotReadyError(RuntimeError): | |
| pass | |
| class Completion: | |
| text: str | |
| input_tokens: int | |
| output_tokens: int | |
| class ModelRuntime: | |
| """Owns the single in-process GGUF model instance.""" | |
| def __init__(self) -> None: | |
| self.llm: Llama | None = None | |
| self.model_path: str | None = None | |
| self.load_error: str | None = None | |
| # A thread lock (not asyncio.Lock) because generation runs in a | |
| # worker thread; a Llama instance is not safe for concurrent use. | |
| self.inference_lock = threading.Lock() | |
| def download_model(self) -> str: | |
| path = hf_hub_download( | |
| repo_id=MODEL_REPO_ID, | |
| filename=MODEL_FILENAME, | |
| repo_type="model", | |
| token=HF_TOKEN or None, | |
| ) | |
| self.model_path = path | |
| return path | |
| def load(self) -> None: | |
| model_path = self.download_model() | |
| self.llm = Llama( | |
| model_path=model_path, | |
| n_ctx=LLAMA_CONTEXT_SIZE, | |
| n_threads=LLAMA_THREADS, | |
| n_gpu_layers=LLAMA_GPU_LAYERS, | |
| verbose=False, | |
| ) | |
| async def start(self) -> None: | |
| """ | |
| Download and load the model once. | |
| A failure is recorded rather than raised so the API still starts | |
| and /health can report the model as not ready. | |
| """ | |
| try: | |
| await asyncio.to_thread(self.load) | |
| self.load_error = None | |
| except Exception as exc: | |
| self.llm = None | |
| self.load_error = f"{type(exc).__name__}: {exc}" | |
| logger.exception("Failed to load the Progression model.") | |
| async def stop(self) -> None: | |
| llm, self.llm = self.llm, None | |
| if llm is not None: | |
| with self.inference_lock: | |
| llm.close() | |
| def ready(self) -> bool: | |
| return self.llm is not None | |
| def _tokenize(self, llm: Llama, text: str, *, prompt: bool) -> list[int]: | |
| # Prompt tokenization mirrors what Llama.create_completion does | |
| # internally (BOS handling + special-token parsing), so the count | |
| # equals what the model actually evaluates. | |
| return llm.tokenize( | |
| text.encode("utf-8"), | |
| add_bos=prompt, | |
| special=prompt, | |
| ) | |
| def _complete(self, prompt: str) -> Completion: | |
| with self.inference_lock: | |
| llm = self.llm | |
| if llm is None: | |
| raise ModelNotReadyError("Progression model is not loaded.") | |
| input_tokens = len(self._tokenize(llm, prompt, prompt=True)) | |
| result = llm.create_completion( | |
| prompt, | |
| max_tokens=MAX_NEW_TOKENS, | |
| temperature=TEMPERATURE, | |
| top_p=TOP_P, | |
| repeat_penalty=REPEAT_PENALTY, | |
| seed=SEED, | |
| echo=False, | |
| ) | |
| text = result["choices"][0]["text"] | |
| output_tokens = len(self._tokenize(llm, text, prompt=False)) | |
| return Completion( | |
| text=text.strip(), | |
| input_tokens=input_tokens, | |
| output_tokens=output_tokens, | |
| ) | |
| async def complete(self, prompt: str) -> Completion: | |
| """ | |
| Raw completion (no chat template) with the internal generation | |
| settings. | |
| The blocking llama.cpp call runs in a worker thread so the event | |
| loop stays responsive; the lock serializes access to the model. | |
| """ | |
| return await asyncio.to_thread(self._complete, prompt) | |
| runtime = ModelRuntime() | |