Spaces:
Sleeping
Sleeping
File size: 3,070 Bytes
ad4bf51 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | """Chooses which inference backend the bot talks to.
Kept separate from both clients so the decision is testable without importing
torch, and so :mod:`finbot.config` stays free of heavy dependencies.
The two backends are interchangeable by design -- same ``enabled`` /
``chat`` / ``chat_json`` / ``vision_json`` surface -- so nothing downstream of
here knows or cares which one it got.
"""
from __future__ import annotations
import logging
from typing import Any
from .config import Settings
log = logging.getLogger(__name__)
BACKENDS = ("auto", "local", "remote", "off")
class NullLLM:
"""No inference at all.
Returned when nothing is configured. Everything deterministic -- typing
expenses, summaries, budgets, rule-based advice -- keeps working; only
receipts, statements and written advice are unavailable.
"""
enabled = False
def describe(self) -> str:
return "disabled (no GPU and no HF_TOKEN)"
def close(self) -> None:
return None
def _fail(self, *_args, **_kwargs):
from .llm import LLMUnavailable
raise LLMUnavailable("no inference backend is configured")
chat = chat_json = vision_json = _fail
def local_available() -> bool:
"""True when a CUDA device is actually usable, not merely installed."""
try:
import torch
return bool(torch.cuda.is_available())
except Exception:
return False
def resolve(settings: Settings, gpu_present: bool | None = None) -> str:
"""Decide the effective backend name. Pure, so it is unit-testable."""
choice = (settings.llm_backend or "auto").strip().lower()
if choice not in BACKENDS:
log.warning("Unknown LLM_BACKEND %r; falling back to auto", choice)
choice = "auto"
if choice != "auto":
return choice
if gpu_present is None:
gpu_present = local_available()
if gpu_present:
return "local"
if settings.hf_token:
return "remote"
return "off"
def make_llm(settings: Settings) -> Any:
"""Build the client for the resolved backend, degrading rather than raising."""
effective = resolve(settings)
if effective == "local":
from .local_llm import LocalLLMClient
client = LocalLLMClient(settings)
if client.enabled:
log.info("Inference backend: local (%s)", client.describe())
return client
# Asked for local but torch is missing -- fall through rather than
# leaving the bot with no inference at all.
log.warning("Local backend requested but unavailable; trying remote.")
effective = "remote" if settings.hf_token else "off"
if effective == "remote":
from .llm import LLMClient
client = LLMClient(settings)
if client.enabled:
log.info("Inference backend: remote (%s)", settings.llm_model)
return client
log.warning("Remote backend requested but HF_TOKEN is unset.")
log.warning("No inference backend available; deterministic features only.")
return NullLLM()
|