File size: 12,828 Bytes
281f5b1
a416f24
281f5b1
 
3712e45
281f5b1
3712e45
281f5b1
 
 
da9de03
 
281f5b1
 
 
 
 
 
 
 
a416f24
281f5b1
a416f24
281f5b1
 
 
 
3712e45
 
 
 
 
 
 
 
 
 
 
30ec081
 
 
 
 
 
a483b94
 
 
 
3712e45
 
 
 
 
 
 
 
a483b94
3712e45
 
 
 
 
 
 
 
 
 
 
a483b94
 
 
 
 
 
 
 
 
 
 
 
 
 
 
da9de03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a483b94
 
 
 
 
 
 
 
 
 
 
3712e45
 
 
 
 
 
 
 
 
da9de03
3712e45
da9de03
 
 
3712e45
 
a483b94
 
 
 
 
 
 
 
 
 
 
3712e45
 
 
da9de03
3712e45
 
30ec081
 
 
 
3712e45
30ec081
3712e45
 
 
 
a483b94
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3712e45
 
a483b94
 
 
 
 
3712e45
 
 
a483b94
 
 
 
 
 
 
 
 
 
3712e45
 
 
 
 
 
 
 
281f5b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a416f24
281f5b1
 
3712e45
 
 
 
 
 
 
 
 
 
 
 
 
 
30ec081
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
# Model clients: one HF InferenceClient for all LLM/VLM calls, and a lazily-loaded
# faster-whisper model for audio/video transcription. No torch - faster-whisper uses
# the ctranslate2 backend and PyAV for decoding.
import os
import random
import threading
import time

from huggingface_hub import InferenceClient

from config import (BREAKER_FAILURES, BREAKER_OPEN_SECONDS, LLM_TIMEOUT, MODEL_PROVIDERS,
                    WHISPER_CPU_THREADS, WHISPER_MODEL)

HF_TOKEN = os.getenv("HF_TOKEN")
if not HF_TOKEN:
    _token_file = os.path.expanduser("~/.cache/huggingface/token")
    if os.path.exists(_token_file):
        HF_TOKEN = open(_token_file).read().strip()

if HF_TOKEN:
    print("[LUNA] HF_TOKEN loaded - inference enabled.")
else:
    print("[LUNA] HF_TOKEN not set - LLM/VLM calls will fail until it is.")

llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)


# ── Retries ───────────────────────────────────────────────────────────────────
# Inference providers rate-limit (429) and cold-start (503) as a matter of course, and
# MAP fires MAP_WORKERS calls at once. Every call used to be a single bare attempt, and
# a failed one is silent: the file degrades to raw text, or drops out of the Note
# altogether. Observed in testing - one flaky call returned unparseable output and the
# single file that answered the question vanished from the Note.
#
# Permanent errors are NOT retried. A bad model name or a revoked token must fail fast
# and loudly rather than be hidden behind three slow attempts.
_RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3"))
_RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5"))
# Ceiling on how long a provider may put us to sleep. Retry-After is the server's
# number, and a build has eight minutes in total: a 429 answered with "come back in
# five minutes" would have parked all eight MAP workers past the deadline and the
# whole task would have timed out waiting politely. Beyond this we stop waiting and
# spend the remaining attempt, or give up and let the file degrade.
_RETRY_MAX_WAIT = float(os.getenv("LUNA_RETRY_MAX_WAIT", "20"))
# "not supported by any provider" is deliberately NOT here: it is a routing verdict,
# not a dead model, and _is_routing handles it. A 402 is: no amount of retrying fixes
# an exhausted balance, and three slow attempts only delay the honest failure.
_PERMANENT = ("invalid_request_error", "authorization", "payment required",
              "unauthorized", "forbidden", "not found", "does not exist")


def _is_permanent(err: Exception) -> bool:
    msg = str(err).lower()
    if any(k in msg for k in _PERMANENT):
        return True
    status = getattr(getattr(err, "response", None), "status_code", None)
    return status in (400, 401, 403, 404, 402)


def _retry_after(err: Exception) -> float | None:
    hdrs = getattr(getattr(err, "response", None), "headers", None) or {}
    try:
        v = hdrs.get("Retry-After") or hdrs.get("retry-after")
        return float(v) if v is not None else None
    except (TypeError, ValueError):
        return None


def _is_routing(err: Exception) -> bool:
    """The router declining to route a model it simply does not list today.

    This is NOT the model being gone. Proven on 2026-09-21: the same name that came
    back "not supported by any provider you have enabled" was served live by three
    providers at that moment, and answered normally when a provider was pinned. It used
    to be classified as permanent, so a build died on the spot with no second attempt.
    Now it means: pin a provider, and failing that, try the next candidate name."""
    return "not supported by any provider" in str(err).lower()


_clients: dict = {}
_clients_lock = threading.Lock()


# ── Circuit breaker ───────────────────────────────────────────────────────────
# Transport failures only (timeouts, 5xx, 429 after its retries, connection errors).
# A permanent error (bad token, bad model name) and a routing refusal are judgements
# about ONE call and do not count. When the provider is down, every one of the eight
# MAP workers used to burn three retries with backoff of its own before the file was
# marked unreadable, and the build still had REDUCE to try; that is minutes of a
# six-minute budget spent learning the same fact eight times.
class ProviderDown(RuntimeError):
    """Raised without a network call while the breaker is open."""


_breaker_lock = threading.Lock()
_breaker_fail = 0
_breaker_open_until = 0.0


def breaker_reset() -> None:
    global _breaker_fail, _breaker_open_until
    with _breaker_lock:
        _breaker_fail, _breaker_open_until = 0, 0.0


def _breaker_check() -> None:
    with _breaker_lock:
        if _breaker_open_until and time.monotonic() < _breaker_open_until:
            raise ProviderDown(f"provider unreachable, retry in "
                               f"{int(_breaker_open_until - time.monotonic()) + 1}s")


def _breaker_ok() -> None:
    global _breaker_fail, _breaker_open_until
    with _breaker_lock:
        _breaker_fail, _breaker_open_until = 0, 0.0


def _breaker_fail_once() -> None:
    global _breaker_fail, _breaker_open_until
    with _breaker_lock:
        _breaker_fail += 1
        if _breaker_fail >= BREAKER_FAILURES:
            if not _breaker_open_until or time.monotonic() >= _breaker_open_until:
                print(f"[LUNA] circuit OPEN: {_breaker_fail} consecutive transport failures, "
                      f"failing fast for {BREAKER_OPEN_SECONDS:.0f}s", flush=True)
            _breaker_open_until = time.monotonic() + BREAKER_OPEN_SECONDS


def _client(provider: str | None):
    """One InferenceClient per pinned provider (None = the auto router)."""
    with _clients_lock:
        if provider not in _clients:
            _clients[provider] = (InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
                                  if provider is None
                                  else InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT,
                                                       provider=provider))
        return _clients[provider]


def chat(**kwargs):
    """`llm_client.chat_completion` with backoff on transient failures.

    Streaming calls are returned on the first successful attempt: once the generator is
    handed out we cannot replay it, so mid-stream failures stay the caller's problem
    (reduce_stream keeps whatever it already received)."""
    label = kwargs.get("model", "?")
    last = None
    for attempt in range(1, _RETRY_ATTEMPTS + 1):
        _breaker_check()
        try:
            out = llm_client.chat_completion(**kwargs)
            _breaker_ok()
            return out
        except Exception as e:
            last = e
            if _is_routing(e):
                # The auto router does not list this model right now. Ask the providers
                # that actually serve it, in order, before giving up on the name.
                for prov in MODEL_PROVIDERS.get(label, []):
                    try:
                        out = _client(prov).chat_completion(**kwargs)
                        print(f"[LUNA] {label}: auto router declined, served by {prov}")
                        return out
                    except Exception as pe:
                        print(f"[LUNA] {label}: provider {prov} failed - {str(pe)[:100]}")
                raise
            if _is_permanent(e):
                print(f"[LUNA] {label}: permanent error, not retrying - {str(e)[:120]}")
                raise
            _breaker_fail_once()
            if attempt == _RETRY_ATTEMPTS:
                break
            asked = _retry_after(e)
            wait = asked if asked is not None else (_RETRY_BASE ** attempt) * (0.7 + random.random() * 0.6)
            wait = max(0.0, min(wait, _RETRY_MAX_WAIT))
            capped = " (capped)" if asked is not None and asked > _RETRY_MAX_WAIT else ""
            print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed "
                  f"({type(e).__name__}), retrying in {wait:.1f}s{capped}")
            time.sleep(wait)
    raise last


def chat_any(models, **kwargs):
    """chat() over a list of candidate names: the first that answers wins.

    A role with one hardcoded name turns any single model's bad day into a failed
    build. Every failure is logged with the name that produced it, so the log says
    which candidate is carrying the work rather than leaving it to be guessed."""
    names = [m for m in dict.fromkeys(models) if m]
    if not names:
        raise RuntimeError("no model configured for this role")
    last = None
    for i, name in enumerate(names, 1):
        try:
            return chat(model=name, **kwargs)
        except Exception as e:
            last = e
            if i < len(names):
                print(f"[LUNA] {name} unusable ({str(e)[:100]}), falling back to {names[i]}")
    raise last


def check_models(names) -> dict:
    """Ask each configured model for one token. A model nobody serves any more is the
    kind of fault that hides for weeks, because every call site degrades quietly.

    Also catches the trap that a plain ping cannot see: a reasoning model answers with
    an empty `content` and its whole budget spent on `reasoning_content`, which at
    REDUCE's budget produces no Note at all and looks exactly like an outage."""
    out = {}
    for n in dict.fromkeys(names):
        try:
            r = chat(model=n, messages=[{"role": "user", "content": "Reply with exactly: ok"}],
                     max_tokens=24, temperature=0)
            msg = r.choices[0].message
            content = (getattr(msg, "content", "") or "").strip()
            thinking = (getattr(msg, "reasoning_content", "") or "").strip()
            if not content and thinking:
                out[n] = False
                print(f"[LUNA] MODEL UNUSABLE: {n} - reasoning model, spends the budget "
                      f"thinking and returns no text ({len(thinking)} thinking chars)")
                continue
            out[n] = True
            print(f"[LUNA] model ok: {n}")
        except Exception as e:
            out[n] = False
            print(f"[LUNA] MODEL UNAVAILABLE: {n} - {str(e)[:140]}")
    return out


# ── Lazy Whisper (faster-whisper, CPU int8) ───────────────────────────────────
_whisper = None
_whisper_lock = threading.Lock()
_whisper_failed = False


def get_whisper():
    """Lazy-load the Whisper model. Returns None (never raises) if it can't load, so
    a task with media degrades gracefully instead of crashing."""
    global _whisper, _whisper_failed
    with _whisper_lock:
        if _whisper_failed:
            return None
        if _whisper is None:
            try:
                from faster_whisper import WhisperModel
                print(f"[LUNA] Loading Whisper {WHISPER_MODEL} (faster-whisper, CPU int8)...")
                _whisper = WhisperModel(
                    WHISPER_MODEL, device="cpu", compute_type="int8",
                    cpu_threads=WHISPER_CPU_THREADS,
                )
                print(f"[LUNA] Whisper {WHISPER_MODEL} ready.")
            except Exception as e:
                _whisper_failed = True
                print(f"[LUNA] Whisper failed to load: {e} - audio/video skipped.")
                return None
    return _whisper


def prewarm_whisper() -> None:
    """Load Whisper in the background at startup.

    It used to load lazily, on the first media file, which put the download and the
    model load straight onto the critical path of a build that the website caps at
    eight minutes. Started here it overlaps with document extraction and MAP, neither
    of which needs it. A task with no media pays nothing either way: the thread is a
    daemon and the model is simply never asked for.
    """
    if os.getenv("LUNA_NO_PREWARM"):
        return
    threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start()


def check_models_async(names) -> None:
    """Run the probe off the startup path.

    It used to run inline at import, three round trips before Gradio began serving,
    each able to sit on LLM_TIMEOUT. One unresponsive provider could therefore hold
    the Space back by nine minutes, on a platform that wakes it from sleep for a task
    already capped at eight. The answer is only ever a log line, so nothing waits on
    it."""
    threading.Thread(target=check_models, args=(list(names),),
                     name="model-probe", daemon=True).start()