# Model clients: one HF InferenceClient for all LLM/VLM calls, and a lazily-loaded # faster-whisper model for audio/video transcription. No torch - faster-whisper uses # the ctranslate2 backend and PyAV for decoding. import os import random import threading import time from huggingface_hub import InferenceClient from config import (BREAKER_FAILURES, BREAKER_OPEN_SECONDS, LLM_TIMEOUT, MODEL_PROVIDERS, WHISPER_CPU_THREADS, WHISPER_MODEL) HF_TOKEN = os.getenv("HF_TOKEN") if not HF_TOKEN: _token_file = os.path.expanduser("~/.cache/huggingface/token") if os.path.exists(_token_file): HF_TOKEN = open(_token_file).read().strip() if HF_TOKEN: print("[LUNA] HF_TOKEN loaded - inference enabled.") else: print("[LUNA] HF_TOKEN not set - LLM/VLM calls will fail until it is.") llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT) # ── Retries ─────────────────────────────────────────────────────────────────── # Inference providers rate-limit (429) and cold-start (503) as a matter of course, and # MAP fires MAP_WORKERS calls at once. Every call used to be a single bare attempt, and # a failed one is silent: the file degrades to raw text, or drops out of the Note # altogether. Observed in testing - one flaky call returned unparseable output and the # single file that answered the question vanished from the Note. # # Permanent errors are NOT retried. A bad model name or a revoked token must fail fast # and loudly rather than be hidden behind three slow attempts. _RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3")) _RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5")) # Ceiling on how long a provider may put us to sleep. Retry-After is the server's # number, and a build has eight minutes in total: a 429 answered with "come back in # five minutes" would have parked all eight MAP workers past the deadline and the # whole task would have timed out waiting politely. Beyond this we stop waiting and # spend the remaining attempt, or give up and let the file degrade. _RETRY_MAX_WAIT = float(os.getenv("LUNA_RETRY_MAX_WAIT", "20")) # "not supported by any provider" is deliberately NOT here: it is a routing verdict, # not a dead model, and _is_routing handles it. A 402 is: no amount of retrying fixes # an exhausted balance, and three slow attempts only delay the honest failure. _PERMANENT = ("invalid_request_error", "authorization", "payment required", "unauthorized", "forbidden", "not found", "does not exist") def _is_permanent(err: Exception) -> bool: msg = str(err).lower() if any(k in msg for k in _PERMANENT): return True status = getattr(getattr(err, "response", None), "status_code", None) return status in (400, 401, 403, 404, 402) def _retry_after(err: Exception) -> float | None: hdrs = getattr(getattr(err, "response", None), "headers", None) or {} try: v = hdrs.get("Retry-After") or hdrs.get("retry-after") return float(v) if v is not None else None except (TypeError, ValueError): return None def _is_routing(err: Exception) -> bool: """The router declining to route a model it simply does not list today. This is NOT the model being gone. Proven on 2026-09-21: the same name that came back "not supported by any provider you have enabled" was served live by three providers at that moment, and answered normally when a provider was pinned. It used to be classified as permanent, so a build died on the spot with no second attempt. Now it means: pin a provider, and failing that, try the next candidate name.""" return "not supported by any provider" in str(err).lower() _clients: dict = {} _clients_lock = threading.Lock() # ── Circuit breaker ─────────────────────────────────────────────────────────── # Transport failures only (timeouts, 5xx, 429 after its retries, connection errors). # A permanent error (bad token, bad model name) and a routing refusal are judgements # about ONE call and do not count. When the provider is down, every one of the eight # MAP workers used to burn three retries with backoff of its own before the file was # marked unreadable, and the build still had REDUCE to try; that is minutes of a # six-minute budget spent learning the same fact eight times. class ProviderDown(RuntimeError): """Raised without a network call while the breaker is open.""" _breaker_lock = threading.Lock() _breaker_fail = 0 _breaker_open_until = 0.0 def breaker_reset() -> None: global _breaker_fail, _breaker_open_until with _breaker_lock: _breaker_fail, _breaker_open_until = 0, 0.0 def _breaker_check() -> None: with _breaker_lock: if _breaker_open_until and time.monotonic() < _breaker_open_until: raise ProviderDown(f"provider unreachable, retry in " f"{int(_breaker_open_until - time.monotonic()) + 1}s") def _breaker_ok() -> None: global _breaker_fail, _breaker_open_until with _breaker_lock: _breaker_fail, _breaker_open_until = 0, 0.0 def _breaker_fail_once() -> None: global _breaker_fail, _breaker_open_until with _breaker_lock: _breaker_fail += 1 if _breaker_fail >= BREAKER_FAILURES: if not _breaker_open_until or time.monotonic() >= _breaker_open_until: print(f"[LUNA] circuit OPEN: {_breaker_fail} consecutive transport failures, " f"failing fast for {BREAKER_OPEN_SECONDS:.0f}s", flush=True) _breaker_open_until = time.monotonic() + BREAKER_OPEN_SECONDS def _client(provider: str | None): """One InferenceClient per pinned provider (None = the auto router).""" with _clients_lock: if provider not in _clients: _clients[provider] = (InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT) if provider is None else InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT, provider=provider)) return _clients[provider] def chat(**kwargs): """`llm_client.chat_completion` with backoff on transient failures. Streaming calls are returned on the first successful attempt: once the generator is handed out we cannot replay it, so mid-stream failures stay the caller's problem (reduce_stream keeps whatever it already received).""" label = kwargs.get("model", "?") last = None for attempt in range(1, _RETRY_ATTEMPTS + 1): _breaker_check() try: out = llm_client.chat_completion(**kwargs) _breaker_ok() return out except Exception as e: last = e if _is_routing(e): # The auto router does not list this model right now. Ask the providers # that actually serve it, in order, before giving up on the name. for prov in MODEL_PROVIDERS.get(label, []): try: out = _client(prov).chat_completion(**kwargs) print(f"[LUNA] {label}: auto router declined, served by {prov}") return out except Exception as pe: print(f"[LUNA] {label}: provider {prov} failed - {str(pe)[:100]}") raise if _is_permanent(e): print(f"[LUNA] {label}: permanent error, not retrying - {str(e)[:120]}") raise _breaker_fail_once() if attempt == _RETRY_ATTEMPTS: break asked = _retry_after(e) wait = asked if asked is not None else (_RETRY_BASE ** attempt) * (0.7 + random.random() * 0.6) wait = max(0.0, min(wait, _RETRY_MAX_WAIT)) capped = " (capped)" if asked is not None and asked > _RETRY_MAX_WAIT else "" print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed " f"({type(e).__name__}), retrying in {wait:.1f}s{capped}") time.sleep(wait) raise last def chat_any(models, **kwargs): """chat() over a list of candidate names: the first that answers wins. A role with one hardcoded name turns any single model's bad day into a failed build. Every failure is logged with the name that produced it, so the log says which candidate is carrying the work rather than leaving it to be guessed.""" names = [m for m in dict.fromkeys(models) if m] if not names: raise RuntimeError("no model configured for this role") last = None for i, name in enumerate(names, 1): try: return chat(model=name, **kwargs) except Exception as e: last = e if i < len(names): print(f"[LUNA] {name} unusable ({str(e)[:100]}), falling back to {names[i]}") raise last def check_models(names) -> dict: """Ask each configured model for one token. A model nobody serves any more is the kind of fault that hides for weeks, because every call site degrades quietly. Also catches the trap that a plain ping cannot see: a reasoning model answers with an empty `content` and its whole budget spent on `reasoning_content`, which at REDUCE's budget produces no Note at all and looks exactly like an outage.""" out = {} for n in dict.fromkeys(names): try: r = chat(model=n, messages=[{"role": "user", "content": "Reply with exactly: ok"}], max_tokens=24, temperature=0) msg = r.choices[0].message content = (getattr(msg, "content", "") or "").strip() thinking = (getattr(msg, "reasoning_content", "") or "").strip() if not content and thinking: out[n] = False print(f"[LUNA] MODEL UNUSABLE: {n} - reasoning model, spends the budget " f"thinking and returns no text ({len(thinking)} thinking chars)") continue out[n] = True print(f"[LUNA] model ok: {n}") except Exception as e: out[n] = False print(f"[LUNA] MODEL UNAVAILABLE: {n} - {str(e)[:140]}") return out # ── Lazy Whisper (faster-whisper, CPU int8) ─────────────────────────────────── _whisper = None _whisper_lock = threading.Lock() _whisper_failed = False def get_whisper(): """Lazy-load the Whisper model. Returns None (never raises) if it can't load, so a task with media degrades gracefully instead of crashing.""" global _whisper, _whisper_failed with _whisper_lock: if _whisper_failed: return None if _whisper is None: try: from faster_whisper import WhisperModel print(f"[LUNA] Loading Whisper {WHISPER_MODEL} (faster-whisper, CPU int8)...") _whisper = WhisperModel( WHISPER_MODEL, device="cpu", compute_type="int8", cpu_threads=WHISPER_CPU_THREADS, ) print(f"[LUNA] Whisper {WHISPER_MODEL} ready.") except Exception as e: _whisper_failed = True print(f"[LUNA] Whisper failed to load: {e} - audio/video skipped.") return None return _whisper def prewarm_whisper() -> None: """Load Whisper in the background at startup. It used to load lazily, on the first media file, which put the download and the model load straight onto the critical path of a build that the website caps at eight minutes. Started here it overlaps with document extraction and MAP, neither of which needs it. A task with no media pays nothing either way: the thread is a daemon and the model is simply never asked for. """ if os.getenv("LUNA_NO_PREWARM"): return threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start() def check_models_async(names) -> None: """Run the probe off the startup path. It used to run inline at import, three round trips before Gradio began serving, each able to sit on LLM_TIMEOUT. One unresponsive provider could therefore hold the Space back by nine minutes, on a platform that wakes it from sleep for a task already capped at eight. The answer is only ever a log line, so nothing waits on it.""" threading.Thread(target=check_models, args=(list(names),), name="model-probe", daemon=True).start()