Spaces:
Paused
Paused
Malina
Claude Fable 5.1
Scoped tokens, bounded archives and frames, build deadline, circuit breaker, honest note caps
da9de03 Download models.py from NextNeural/LUNA: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/models.py
- Command line
-
hf download hf://spaces/NextNeural/LUNA/models.py
-
curl -L -o models.py https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/models.py
12.8 kB
| # Model clients: one HF InferenceClient for all LLM/VLM calls, and a lazily-loaded | |
| # faster-whisper model for audio/video transcription. No torch - faster-whisper uses | |
| # the ctranslate2 backend and PyAV for decoding. | |
| import os | |
| import random | |
| import threading | |
| import time | |
| from huggingface_hub import InferenceClient | |
| from config import (BREAKER_FAILURES, BREAKER_OPEN_SECONDS, LLM_TIMEOUT, MODEL_PROVIDERS, | |
| WHISPER_CPU_THREADS, WHISPER_MODEL) | |
| HF_TOKEN = os.getenv("HF_TOKEN") | |
| if not HF_TOKEN: | |
| _token_file = os.path.expanduser("~/.cache/huggingface/token") | |
| if os.path.exists(_token_file): | |
| HF_TOKEN = open(_token_file).read().strip() | |
| if HF_TOKEN: | |
| print("[LUNA] HF_TOKEN loaded - inference enabled.") | |
| else: | |
| print("[LUNA] HF_TOKEN not set - LLM/VLM calls will fail until it is.") | |
| llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT) | |
| # ββ Retries βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Inference providers rate-limit (429) and cold-start (503) as a matter of course, and | |
| # MAP fires MAP_WORKERS calls at once. Every call used to be a single bare attempt, and | |
| # a failed one is silent: the file degrades to raw text, or drops out of the Note | |
| # altogether. Observed in testing - one flaky call returned unparseable output and the | |
| # single file that answered the question vanished from the Note. | |
| # | |
| # Permanent errors are NOT retried. A bad model name or a revoked token must fail fast | |
| # and loudly rather than be hidden behind three slow attempts. | |
| _RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3")) | |
| _RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5")) | |
| # Ceiling on how long a provider may put us to sleep. Retry-After is the server's | |
| # number, and a build has eight minutes in total: a 429 answered with "come back in | |
| # five minutes" would have parked all eight MAP workers past the deadline and the | |
| # whole task would have timed out waiting politely. Beyond this we stop waiting and | |
| # spend the remaining attempt, or give up and let the file degrade. | |
| _RETRY_MAX_WAIT = float(os.getenv("LUNA_RETRY_MAX_WAIT", "20")) | |
| # "not supported by any provider" is deliberately NOT here: it is a routing verdict, | |
| # not a dead model, and _is_routing handles it. A 402 is: no amount of retrying fixes | |
| # an exhausted balance, and three slow attempts only delay the honest failure. | |
| _PERMANENT = ("invalid_request_error", "authorization", "payment required", | |
| "unauthorized", "forbidden", "not found", "does not exist") | |
| def _is_permanent(err: Exception) -> bool: | |
| msg = str(err).lower() | |
| if any(k in msg for k in _PERMANENT): | |
| return True | |
| status = getattr(getattr(err, "response", None), "status_code", None) | |
| return status in (400, 401, 403, 404, 402) | |
| def _retry_after(err: Exception) -> float | None: | |
| hdrs = getattr(getattr(err, "response", None), "headers", None) or {} | |
| try: | |
| v = hdrs.get("Retry-After") or hdrs.get("retry-after") | |
| return float(v) if v is not None else None | |
| except (TypeError, ValueError): | |
| return None | |
| def _is_routing(err: Exception) -> bool: | |
| """The router declining to route a model it simply does not list today. | |
| This is NOT the model being gone. Proven on 2026-09-21: the same name that came | |
| back "not supported by any provider you have enabled" was served live by three | |
| providers at that moment, and answered normally when a provider was pinned. It used | |
| to be classified as permanent, so a build died on the spot with no second attempt. | |
| Now it means: pin a provider, and failing that, try the next candidate name.""" | |
| return "not supported by any provider" in str(err).lower() | |
| _clients: dict = {} | |
| _clients_lock = threading.Lock() | |
| # ββ Circuit breaker βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Transport failures only (timeouts, 5xx, 429 after its retries, connection errors). | |
| # A permanent error (bad token, bad model name) and a routing refusal are judgements | |
| # about ONE call and do not count. When the provider is down, every one of the eight | |
| # MAP workers used to burn three retries with backoff of its own before the file was | |
| # marked unreadable, and the build still had REDUCE to try; that is minutes of a | |
| # six-minute budget spent learning the same fact eight times. | |
| class ProviderDown(RuntimeError): | |
| """Raised without a network call while the breaker is open.""" | |
| _breaker_lock = threading.Lock() | |
| _breaker_fail = 0 | |
| _breaker_open_until = 0.0 | |
| def breaker_reset() -> None: | |
| global _breaker_fail, _breaker_open_until | |
| with _breaker_lock: | |
| _breaker_fail, _breaker_open_until = 0, 0.0 | |
| def _breaker_check() -> None: | |
| with _breaker_lock: | |
| if _breaker_open_until and time.monotonic() < _breaker_open_until: | |
| raise ProviderDown(f"provider unreachable, retry in " | |
| f"{int(_breaker_open_until - time.monotonic()) + 1}s") | |
| def _breaker_ok() -> None: | |
| global _breaker_fail, _breaker_open_until | |
| with _breaker_lock: | |
| _breaker_fail, _breaker_open_until = 0, 0.0 | |
| def _breaker_fail_once() -> None: | |
| global _breaker_fail, _breaker_open_until | |
| with _breaker_lock: | |
| _breaker_fail += 1 | |
| if _breaker_fail >= BREAKER_FAILURES: | |
| if not _breaker_open_until or time.monotonic() >= _breaker_open_until: | |
| print(f"[LUNA] circuit OPEN: {_breaker_fail} consecutive transport failures, " | |
| f"failing fast for {BREAKER_OPEN_SECONDS:.0f}s", flush=True) | |
| _breaker_open_until = time.monotonic() + BREAKER_OPEN_SECONDS | |
| def _client(provider: str | None): | |
| """One InferenceClient per pinned provider (None = the auto router).""" | |
| with _clients_lock: | |
| if provider not in _clients: | |
| _clients[provider] = (InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT) | |
| if provider is None | |
| else InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT, | |
| provider=provider)) | |
| return _clients[provider] | |
| def chat(**kwargs): | |
| """`llm_client.chat_completion` with backoff on transient failures. | |
| Streaming calls are returned on the first successful attempt: once the generator is | |
| handed out we cannot replay it, so mid-stream failures stay the caller's problem | |
| (reduce_stream keeps whatever it already received).""" | |
| label = kwargs.get("model", "?") | |
| last = None | |
| for attempt in range(1, _RETRY_ATTEMPTS + 1): | |
| _breaker_check() | |
| try: | |
| out = llm_client.chat_completion(**kwargs) | |
| _breaker_ok() | |
| return out | |
| except Exception as e: | |
| last = e | |
| if _is_routing(e): | |
| # The auto router does not list this model right now. Ask the providers | |
| # that actually serve it, in order, before giving up on the name. | |
| for prov in MODEL_PROVIDERS.get(label, []): | |
| try: | |
| out = _client(prov).chat_completion(**kwargs) | |
| print(f"[LUNA] {label}: auto router declined, served by {prov}") | |
| return out | |
| except Exception as pe: | |
| print(f"[LUNA] {label}: provider {prov} failed - {str(pe)[:100]}") | |
| raise | |
| if _is_permanent(e): | |
| print(f"[LUNA] {label}: permanent error, not retrying - {str(e)[:120]}") | |
| raise | |
| _breaker_fail_once() | |
| if attempt == _RETRY_ATTEMPTS: | |
| break | |
| asked = _retry_after(e) | |
| wait = asked if asked is not None else (_RETRY_BASE ** attempt) * (0.7 + random.random() * 0.6) | |
| wait = max(0.0, min(wait, _RETRY_MAX_WAIT)) | |
| capped = " (capped)" if asked is not None and asked > _RETRY_MAX_WAIT else "" | |
| print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed " | |
| f"({type(e).__name__}), retrying in {wait:.1f}s{capped}") | |
| time.sleep(wait) | |
| raise last | |
| def chat_any(models, **kwargs): | |
| """chat() over a list of candidate names: the first that answers wins. | |
| A role with one hardcoded name turns any single model's bad day into a failed | |
| build. Every failure is logged with the name that produced it, so the log says | |
| which candidate is carrying the work rather than leaving it to be guessed.""" | |
| names = [m for m in dict.fromkeys(models) if m] | |
| if not names: | |
| raise RuntimeError("no model configured for this role") | |
| last = None | |
| for i, name in enumerate(names, 1): | |
| try: | |
| return chat(model=name, **kwargs) | |
| except Exception as e: | |
| last = e | |
| if i < len(names): | |
| print(f"[LUNA] {name} unusable ({str(e)[:100]}), falling back to {names[i]}") | |
| raise last | |
| def check_models(names) -> dict: | |
| """Ask each configured model for one token. A model nobody serves any more is the | |
| kind of fault that hides for weeks, because every call site degrades quietly. | |
| Also catches the trap that a plain ping cannot see: a reasoning model answers with | |
| an empty `content` and its whole budget spent on `reasoning_content`, which at | |
| REDUCE's budget produces no Note at all and looks exactly like an outage.""" | |
| out = {} | |
| for n in dict.fromkeys(names): | |
| try: | |
| r = chat(model=n, messages=[{"role": "user", "content": "Reply with exactly: ok"}], | |
| max_tokens=24, temperature=0) | |
| msg = r.choices[0].message | |
| content = (getattr(msg, "content", "") or "").strip() | |
| thinking = (getattr(msg, "reasoning_content", "") or "").strip() | |
| if not content and thinking: | |
| out[n] = False | |
| print(f"[LUNA] MODEL UNUSABLE: {n} - reasoning model, spends the budget " | |
| f"thinking and returns no text ({len(thinking)} thinking chars)") | |
| continue | |
| out[n] = True | |
| print(f"[LUNA] model ok: {n}") | |
| except Exception as e: | |
| out[n] = False | |
| print(f"[LUNA] MODEL UNAVAILABLE: {n} - {str(e)[:140]}") | |
| return out | |
| # ββ Lazy Whisper (faster-whisper, CPU int8) βββββββββββββββββββββββββββββββββββ | |
| _whisper = None | |
| _whisper_lock = threading.Lock() | |
| _whisper_failed = False | |
| def get_whisper(): | |
| """Lazy-load the Whisper model. Returns None (never raises) if it can't load, so | |
| a task with media degrades gracefully instead of crashing.""" | |
| global _whisper, _whisper_failed | |
| with _whisper_lock: | |
| if _whisper_failed: | |
| return None | |
| if _whisper is None: | |
| try: | |
| from faster_whisper import WhisperModel | |
| print(f"[LUNA] Loading Whisper {WHISPER_MODEL} (faster-whisper, CPU int8)...") | |
| _whisper = WhisperModel( | |
| WHISPER_MODEL, device="cpu", compute_type="int8", | |
| cpu_threads=WHISPER_CPU_THREADS, | |
| ) | |
| print(f"[LUNA] Whisper {WHISPER_MODEL} ready.") | |
| except Exception as e: | |
| _whisper_failed = True | |
| print(f"[LUNA] Whisper failed to load: {e} - audio/video skipped.") | |
| return None | |
| return _whisper | |
| def prewarm_whisper() -> None: | |
| """Load Whisper in the background at startup. | |
| It used to load lazily, on the first media file, which put the download and the | |
| model load straight onto the critical path of a build that the website caps at | |
| eight minutes. Started here it overlaps with document extraction and MAP, neither | |
| of which needs it. A task with no media pays nothing either way: the thread is a | |
| daemon and the model is simply never asked for. | |
| """ | |
| if os.getenv("LUNA_NO_PREWARM"): | |
| return | |
| threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start() | |
| def check_models_async(names) -> None: | |
| """Run the probe off the startup path. | |
| It used to run inline at import, three round trips before Gradio began serving, | |
| each able to sit on LLM_TIMEOUT. One unresponsive provider could therefore hold | |
| the Space back by nine minutes, on a platform that wakes it from sleep for a task | |
| already capped at eight. The answer is only ever a log line, so nothing waits on | |
| it.""" | |
| threading.Thread(target=check_models, args=(list(names),), | |
| name="model-probe", daemon=True).start() | |