File size: 1,272 Bytes
c63bc31 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Model adapters: a single interface over different inference backends.
The engine never talks to a model directly. It talks to a ``ModelAdapter``.
That keeps the agent loop testable without a GPU and lets the same code drive
a mock, a small local model, or the full 1.65T checkpoint.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any
@dataclass
class GenerationResult:
"""Raw text produced by a model, plus usage counters."""
text: str
prompt_tokens: int = 0
completion_tokens: int = 0
class ModelAdapter(ABC):
"""Interface every backend must implement."""
#: Identifier reported by the API, e.g. ``Frankenstein-Labs/Cortex-ai``.
model_id: str = "unknown"
@abstractmethod
def generate(
self,
prompt: str,
*,
max_new_tokens: int = 512,
temperature: float = 0.6,
) -> GenerationResult:
"""Run one completion over an already-encoded prompt."""
def count_tokens(self, text: str) -> int:
"""Approximate token count. Backends with a tokenizer should override."""
return len(text.split())
def close(self) -> None:
"""Release resources. Default: nothing to do."""
|