Spaces:
Running on Zero
Running on Zero
| """Chooses which inference backend the bot talks to. | |
| Kept separate from both clients so the decision is testable without importing | |
| torch, and so :mod:`finbot.config` stays free of heavy dependencies. | |
| The two backends are interchangeable by design -- same ``enabled`` / | |
| ``chat`` / ``chat_json`` / ``vision_json`` surface -- so nothing downstream of | |
| here knows or cares which one it got. | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| from typing import Any | |
| from .config import Settings | |
| log = logging.getLogger(__name__) | |
| BACKENDS = ("auto", "local", "remote", "off") | |
| class NullLLM: | |
| """No inference at all. | |
| Returned when nothing is configured. Everything deterministic -- typing | |
| expenses, summaries, budgets, rule-based advice -- keeps working; only | |
| receipts, statements and written advice are unavailable. | |
| """ | |
| enabled = False | |
| def describe(self) -> str: | |
| return "disabled (no GPU and no HF_TOKEN)" | |
| def close(self) -> None: | |
| return None | |
| def _fail(self, *_args, **_kwargs): | |
| from .llm import LLMUnavailable | |
| raise LLMUnavailable("no inference backend is configured") | |
| chat = chat_json = vision_json = _fail | |
| def local_available() -> bool: | |
| """True when a CUDA device is actually usable, not merely installed.""" | |
| try: | |
| import torch | |
| return bool(torch.cuda.is_available()) | |
| except Exception: | |
| return False | |
| def resolve(settings: Settings, gpu_present: bool | None = None) -> str: | |
| """Decide the effective backend name. Pure, so it is unit-testable.""" | |
| choice = (settings.llm_backend or "auto").strip().lower() | |
| if choice not in BACKENDS: | |
| log.warning("Unknown LLM_BACKEND %r; falling back to auto", choice) | |
| choice = "auto" | |
| if choice != "auto": | |
| return choice | |
| if gpu_present is None: | |
| gpu_present = local_available() | |
| if gpu_present: | |
| return "local" | |
| if settings.hf_token: | |
| return "remote" | |
| return "off" | |
| def make_llm(settings: Settings) -> Any: | |
| """Build the client for the resolved backend, degrading rather than raising.""" | |
| effective = resolve(settings) | |
| if effective == "local": | |
| from .local_llm import LocalLLMClient | |
| client = LocalLLMClient(settings) | |
| if client.enabled: | |
| log.info("Inference backend: local (%s)", client.describe()) | |
| return client | |
| # Asked for local but torch is missing -- fall through rather than | |
| # leaving the bot with no inference at all. | |
| log.warning("Local backend requested but unavailable; trying remote.") | |
| effective = "remote" if settings.hf_token else "off" | |
| if effective == "remote": | |
| from .llm import LLMClient | |
| client = LLMClient(settings) | |
| if client.enabled: | |
| log.info("Inference backend: remote (%s)", settings.llm_model) | |
| return client | |
| log.warning("Remote backend requested but HF_TOKEN is unset.") | |
| log.warning("No inference backend available; deterministic features only.") | |
| return NullLLM() | |