Spaces:
Paused
Paused
File size: 12,828 Bytes
281f5b1 a416f24 281f5b1 3712e45 281f5b1 3712e45 281f5b1 da9de03 281f5b1 a416f24 281f5b1 a416f24 281f5b1 3712e45 30ec081 a483b94 3712e45 a483b94 3712e45 a483b94 da9de03 a483b94 3712e45 da9de03 3712e45 da9de03 3712e45 a483b94 3712e45 da9de03 3712e45 30ec081 3712e45 30ec081 3712e45 a483b94 3712e45 a483b94 3712e45 a483b94 3712e45 281f5b1 a416f24 281f5b1 3712e45 30ec081 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 | # Model clients: one HF InferenceClient for all LLM/VLM calls, and a lazily-loaded
# faster-whisper model for audio/video transcription. No torch - faster-whisper uses
# the ctranslate2 backend and PyAV for decoding.
import os
import random
import threading
import time
from huggingface_hub import InferenceClient
from config import (BREAKER_FAILURES, BREAKER_OPEN_SECONDS, LLM_TIMEOUT, MODEL_PROVIDERS,
WHISPER_CPU_THREADS, WHISPER_MODEL)
HF_TOKEN = os.getenv("HF_TOKEN")
if not HF_TOKEN:
_token_file = os.path.expanduser("~/.cache/huggingface/token")
if os.path.exists(_token_file):
HF_TOKEN = open(_token_file).read().strip()
if HF_TOKEN:
print("[LUNA] HF_TOKEN loaded - inference enabled.")
else:
print("[LUNA] HF_TOKEN not set - LLM/VLM calls will fail until it is.")
llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
# ββ Retries βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Inference providers rate-limit (429) and cold-start (503) as a matter of course, and
# MAP fires MAP_WORKERS calls at once. Every call used to be a single bare attempt, and
# a failed one is silent: the file degrades to raw text, or drops out of the Note
# altogether. Observed in testing - one flaky call returned unparseable output and the
# single file that answered the question vanished from the Note.
#
# Permanent errors are NOT retried. A bad model name or a revoked token must fail fast
# and loudly rather than be hidden behind three slow attempts.
_RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3"))
_RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5"))
# Ceiling on how long a provider may put us to sleep. Retry-After is the server's
# number, and a build has eight minutes in total: a 429 answered with "come back in
# five minutes" would have parked all eight MAP workers past the deadline and the
# whole task would have timed out waiting politely. Beyond this we stop waiting and
# spend the remaining attempt, or give up and let the file degrade.
_RETRY_MAX_WAIT = float(os.getenv("LUNA_RETRY_MAX_WAIT", "20"))
# "not supported by any provider" is deliberately NOT here: it is a routing verdict,
# not a dead model, and _is_routing handles it. A 402 is: no amount of retrying fixes
# an exhausted balance, and three slow attempts only delay the honest failure.
_PERMANENT = ("invalid_request_error", "authorization", "payment required",
"unauthorized", "forbidden", "not found", "does not exist")
def _is_permanent(err: Exception) -> bool:
msg = str(err).lower()
if any(k in msg for k in _PERMANENT):
return True
status = getattr(getattr(err, "response", None), "status_code", None)
return status in (400, 401, 403, 404, 402)
def _retry_after(err: Exception) -> float | None:
hdrs = getattr(getattr(err, "response", None), "headers", None) or {}
try:
v = hdrs.get("Retry-After") or hdrs.get("retry-after")
return float(v) if v is not None else None
except (TypeError, ValueError):
return None
def _is_routing(err: Exception) -> bool:
"""The router declining to route a model it simply does not list today.
This is NOT the model being gone. Proven on 2026-09-21: the same name that came
back "not supported by any provider you have enabled" was served live by three
providers at that moment, and answered normally when a provider was pinned. It used
to be classified as permanent, so a build died on the spot with no second attempt.
Now it means: pin a provider, and failing that, try the next candidate name."""
return "not supported by any provider" in str(err).lower()
_clients: dict = {}
_clients_lock = threading.Lock()
# ββ Circuit breaker βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Transport failures only (timeouts, 5xx, 429 after its retries, connection errors).
# A permanent error (bad token, bad model name) and a routing refusal are judgements
# about ONE call and do not count. When the provider is down, every one of the eight
# MAP workers used to burn three retries with backoff of its own before the file was
# marked unreadable, and the build still had REDUCE to try; that is minutes of a
# six-minute budget spent learning the same fact eight times.
class ProviderDown(RuntimeError):
"""Raised without a network call while the breaker is open."""
_breaker_lock = threading.Lock()
_breaker_fail = 0
_breaker_open_until = 0.0
def breaker_reset() -> None:
global _breaker_fail, _breaker_open_until
with _breaker_lock:
_breaker_fail, _breaker_open_until = 0, 0.0
def _breaker_check() -> None:
with _breaker_lock:
if _breaker_open_until and time.monotonic() < _breaker_open_until:
raise ProviderDown(f"provider unreachable, retry in "
f"{int(_breaker_open_until - time.monotonic()) + 1}s")
def _breaker_ok() -> None:
global _breaker_fail, _breaker_open_until
with _breaker_lock:
_breaker_fail, _breaker_open_until = 0, 0.0
def _breaker_fail_once() -> None:
global _breaker_fail, _breaker_open_until
with _breaker_lock:
_breaker_fail += 1
if _breaker_fail >= BREAKER_FAILURES:
if not _breaker_open_until or time.monotonic() >= _breaker_open_until:
print(f"[LUNA] circuit OPEN: {_breaker_fail} consecutive transport failures, "
f"failing fast for {BREAKER_OPEN_SECONDS:.0f}s", flush=True)
_breaker_open_until = time.monotonic() + BREAKER_OPEN_SECONDS
def _client(provider: str | None):
"""One InferenceClient per pinned provider (None = the auto router)."""
with _clients_lock:
if provider not in _clients:
_clients[provider] = (InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
if provider is None
else InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT,
provider=provider))
return _clients[provider]
def chat(**kwargs):
"""`llm_client.chat_completion` with backoff on transient failures.
Streaming calls are returned on the first successful attempt: once the generator is
handed out we cannot replay it, so mid-stream failures stay the caller's problem
(reduce_stream keeps whatever it already received)."""
label = kwargs.get("model", "?")
last = None
for attempt in range(1, _RETRY_ATTEMPTS + 1):
_breaker_check()
try:
out = llm_client.chat_completion(**kwargs)
_breaker_ok()
return out
except Exception as e:
last = e
if _is_routing(e):
# The auto router does not list this model right now. Ask the providers
# that actually serve it, in order, before giving up on the name.
for prov in MODEL_PROVIDERS.get(label, []):
try:
out = _client(prov).chat_completion(**kwargs)
print(f"[LUNA] {label}: auto router declined, served by {prov}")
return out
except Exception as pe:
print(f"[LUNA] {label}: provider {prov} failed - {str(pe)[:100]}")
raise
if _is_permanent(e):
print(f"[LUNA] {label}: permanent error, not retrying - {str(e)[:120]}")
raise
_breaker_fail_once()
if attempt == _RETRY_ATTEMPTS:
break
asked = _retry_after(e)
wait = asked if asked is not None else (_RETRY_BASE ** attempt) * (0.7 + random.random() * 0.6)
wait = max(0.0, min(wait, _RETRY_MAX_WAIT))
capped = " (capped)" if asked is not None and asked > _RETRY_MAX_WAIT else ""
print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed "
f"({type(e).__name__}), retrying in {wait:.1f}s{capped}")
time.sleep(wait)
raise last
def chat_any(models, **kwargs):
"""chat() over a list of candidate names: the first that answers wins.
A role with one hardcoded name turns any single model's bad day into a failed
build. Every failure is logged with the name that produced it, so the log says
which candidate is carrying the work rather than leaving it to be guessed."""
names = [m for m in dict.fromkeys(models) if m]
if not names:
raise RuntimeError("no model configured for this role")
last = None
for i, name in enumerate(names, 1):
try:
return chat(model=name, **kwargs)
except Exception as e:
last = e
if i < len(names):
print(f"[LUNA] {name} unusable ({str(e)[:100]}), falling back to {names[i]}")
raise last
def check_models(names) -> dict:
"""Ask each configured model for one token. A model nobody serves any more is the
kind of fault that hides for weeks, because every call site degrades quietly.
Also catches the trap that a plain ping cannot see: a reasoning model answers with
an empty `content` and its whole budget spent on `reasoning_content`, which at
REDUCE's budget produces no Note at all and looks exactly like an outage."""
out = {}
for n in dict.fromkeys(names):
try:
r = chat(model=n, messages=[{"role": "user", "content": "Reply with exactly: ok"}],
max_tokens=24, temperature=0)
msg = r.choices[0].message
content = (getattr(msg, "content", "") or "").strip()
thinking = (getattr(msg, "reasoning_content", "") or "").strip()
if not content and thinking:
out[n] = False
print(f"[LUNA] MODEL UNUSABLE: {n} - reasoning model, spends the budget "
f"thinking and returns no text ({len(thinking)} thinking chars)")
continue
out[n] = True
print(f"[LUNA] model ok: {n}")
except Exception as e:
out[n] = False
print(f"[LUNA] MODEL UNAVAILABLE: {n} - {str(e)[:140]}")
return out
# ββ Lazy Whisper (faster-whisper, CPU int8) βββββββββββββββββββββββββββββββββββ
_whisper = None
_whisper_lock = threading.Lock()
_whisper_failed = False
def get_whisper():
"""Lazy-load the Whisper model. Returns None (never raises) if it can't load, so
a task with media degrades gracefully instead of crashing."""
global _whisper, _whisper_failed
with _whisper_lock:
if _whisper_failed:
return None
if _whisper is None:
try:
from faster_whisper import WhisperModel
print(f"[LUNA] Loading Whisper {WHISPER_MODEL} (faster-whisper, CPU int8)...")
_whisper = WhisperModel(
WHISPER_MODEL, device="cpu", compute_type="int8",
cpu_threads=WHISPER_CPU_THREADS,
)
print(f"[LUNA] Whisper {WHISPER_MODEL} ready.")
except Exception as e:
_whisper_failed = True
print(f"[LUNA] Whisper failed to load: {e} - audio/video skipped.")
return None
return _whisper
def prewarm_whisper() -> None:
"""Load Whisper in the background at startup.
It used to load lazily, on the first media file, which put the download and the
model load straight onto the critical path of a build that the website caps at
eight minutes. Started here it overlaps with document extraction and MAP, neither
of which needs it. A task with no media pays nothing either way: the thread is a
daemon and the model is simply never asked for.
"""
if os.getenv("LUNA_NO_PREWARM"):
return
threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start()
def check_models_async(names) -> None:
"""Run the probe off the startup path.
It used to run inline at import, three round trips before Gradio began serving,
each able to sit on LLM_TIMEOUT. One unresponsive provider could therefore hold
the Space back by nine minutes, on a platform that wakes it from sleep for a task
already capped at eight. The answer is only ever a log line, so nothing waits on
it."""
threading.Thread(target=check_models, args=(list(names),),
name="model-probe", daemon=True).start()
|