File size: 4,025 Bytes
63d4e3a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
from __future__ import annotations

import asyncio
import logging
import threading
from dataclasses import dataclass

from huggingface_hub import hf_hub_download
from llama_cpp import Llama

from app.config import (
    HF_TOKEN,
    LLAMA_CONTEXT_SIZE,
    LLAMA_GPU_LAYERS,
    LLAMA_THREADS,
    MAX_NEW_TOKENS,
    MODEL_FILENAME,
    MODEL_REPO_ID,
    REPEAT_PENALTY,
    SEED,
    TEMPERATURE,
    TOP_P,
)

logger = logging.getLogger(__name__)


class ModelNotReadyError(RuntimeError):
    pass


@dataclass(frozen=True)
class Completion:
    text: str
    input_tokens: int
    output_tokens: int


class ModelRuntime:
    """Owns the single in-process GGUF model instance."""

    def __init__(self) -> None:
        self.llm: Llama | None = None
        self.model_path: str | None = None
        self.load_error: str | None = None
        # A thread lock (not asyncio.Lock) because generation runs in a
        # worker thread; a Llama instance is not safe for concurrent use.
        self.inference_lock = threading.Lock()

    def download_model(self) -> str:
        path = hf_hub_download(
            repo_id=MODEL_REPO_ID,
            filename=MODEL_FILENAME,
            repo_type="model",
            token=HF_TOKEN or None,
        )

        self.model_path = path
        return path

    def load(self) -> None:
        model_path = self.download_model()

        self.llm = Llama(
            model_path=model_path,
            n_ctx=LLAMA_CONTEXT_SIZE,
            n_threads=LLAMA_THREADS,
            n_gpu_layers=LLAMA_GPU_LAYERS,
            verbose=False,
        )

    async def start(self) -> None:
        """
        Download and load the model once.

        A failure is recorded rather than raised so the API still starts
        and /health can report the model as not ready.
        """
        try:
            await asyncio.to_thread(self.load)
            self.load_error = None
        except Exception as exc:
            self.llm = None
            self.load_error = f"{type(exc).__name__}: {exc}"
            logger.exception("Failed to load the Progression model.")

    async def stop(self) -> None:
        llm, self.llm = self.llm, None

        if llm is not None:
            with self.inference_lock:
                llm.close()

    def ready(self) -> bool:
        return self.llm is not None

    def _tokenize(self, llm: Llama, text: str, *, prompt: bool) -> list[int]:
        # Prompt tokenization mirrors what Llama.create_completion does
        # internally (BOS handling + special-token parsing), so the count
        # equals what the model actually evaluates.
        return llm.tokenize(
            text.encode("utf-8"),
            add_bos=prompt,
            special=prompt,
        )

    def _complete(self, prompt: str) -> Completion:
        with self.inference_lock:
            llm = self.llm

            if llm is None:
                raise ModelNotReadyError("Progression model is not loaded.")

            input_tokens = len(self._tokenize(llm, prompt, prompt=True))

            result = llm.create_completion(
                prompt,
                max_tokens=MAX_NEW_TOKENS,
                temperature=TEMPERATURE,
                top_p=TOP_P,
                repeat_penalty=REPEAT_PENALTY,
                seed=SEED,
                echo=False,
            )

            text = result["choices"][0]["text"]
            output_tokens = len(self._tokenize(llm, text, prompt=False))

        return Completion(
            text=text.strip(),
            input_tokens=input_tokens,
            output_tokens=output_tokens,
        )

    async def complete(self, prompt: str) -> Completion:
        """
        Raw completion (no chat template) with the internal generation
        settings.

        The blocking llama.cpp call runs in a worker thread so the event
        loop stays responsive; the lock serializes access to the model.
        """
        return await asyncio.to_thread(self._complete, prompt)


runtime = ModelRuntime()