Spaces:
Paused
Dostawca nie moze uspic nas na dluzej, niz trwa cale zadanie
Browse filesDwie rzeczy, ktore przy zlym dniu dostawcy inferencji koncza sie zadaniem
przeterminowanym bez jednej linijki Notatki.
1. Ponawianie honorowalo naglowek Retry-After bez zadnego ograniczenia. To
liczba serwera, a build ma osiem minut w sumie: odpowiedz "wroc za piec
minut" na 429 zaparkowalaby wszystkie osiem watkow MAP za linia konca i
cale zadanie przepadloby, czekajac grzecznie. Czas oczekiwania jest teraz
ograniczony do 20 sekund (LUNA_RETRY_MAX_WAIT), a log mowi, kiedy przyciete.
2. check_models wolalo trzy modele synchronicznie przy imporcie, czyli zanim
Gradio zaczelo obslugiwac. Kazde z tych wywolan moze siedziec na
LLM_TIMEOUT, ktory wynosi 180 s, wiec jeden niereagujacy dostawca mogl
wstrzymac wstanie Space'u o dziewiec minut - na platformie, ktora budzi go
ze snu do zadania i tak ograniczonego do osmiu. Sonda idzie teraz w tlo,
bo jej wynikiem jest wylacznie linijka w logu i nic na nia nie czeka.
Testy na oba, plus na to, ze blad trwaly (uniewazniony token) nie jest
ponawiany ani razu. Przy okazji test na sam backoff musial sprawdzac przedzial,
nie kolejnosc: jitter jest celowy, zeby osiem watkow nie wrocilo naraz.
Testy: 47 -> 52.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
- app.py +2 -2
- models.py +23 -2
- test_luna.py +80 -0
|
@@ -9,7 +9,7 @@ import gradio as gr
|
|
| 9 |
|
| 10 |
from auth import verify_usage_token
|
| 11 |
from config import MAP_MODEL, REDUCE_MODEL, VLM_MODEL
|
| 12 |
-
from models import
|
| 13 |
from pipeline import run
|
| 14 |
from synthesis import clarify as _clarify
|
| 15 |
|
|
@@ -24,7 +24,7 @@ def clarify(prompt, token):
|
|
| 24 |
# Say out loud, once, whether the configured models are actually served. A name that
|
| 25 |
# no provider carries any more fails every call, and every call site degrades quietly,
|
| 26 |
# so the whole extraction stage can be dead for weeks without a single visible error.
|
| 27 |
-
|
| 28 |
prewarm_whisper()
|
| 29 |
|
| 30 |
|
|
|
|
| 9 |
|
| 10 |
from auth import verify_usage_token
|
| 11 |
from config import MAP_MODEL, REDUCE_MODEL, VLM_MODEL
|
| 12 |
+
from models import check_models_async, prewarm_whisper
|
| 13 |
from pipeline import run
|
| 14 |
from synthesis import clarify as _clarify
|
| 15 |
|
|
|
|
| 24 |
# Say out loud, once, whether the configured models are actually served. A name that
|
| 25 |
# no provider carries any more fails every call, and every call site degrades quietly,
|
| 26 |
# so the whole extraction stage can be dead for weeks without a single visible error.
|
| 27 |
+
check_models_async([MAP_MODEL, REDUCE_MODEL, VLM_MODEL])
|
| 28 |
prewarm_whisper()
|
| 29 |
|
| 30 |
|
|
@@ -35,6 +35,12 @@ llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
|
|
| 35 |
# and loudly rather than be hidden behind three slow attempts.
|
| 36 |
_RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3"))
|
| 37 |
_RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
_PERMANENT = ("not supported by any provider", "invalid_request_error", "authorization",
|
| 39 |
"unauthorized", "forbidden", "not found", "does not exist")
|
| 40 |
|
|
@@ -74,9 +80,12 @@ def chat(**kwargs):
|
|
| 74 |
raise
|
| 75 |
if attempt == _RETRY_ATTEMPTS:
|
| 76 |
break
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
| 78 |
print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed "
|
| 79 |
-
f"({type(e).__name__}), retrying in {wait:.1f}s")
|
| 80 |
time.sleep(wait)
|
| 81 |
raise last
|
| 82 |
|
|
@@ -138,3 +147,15 @@ def prewarm_whisper() -> None:
|
|
| 138 |
if os.getenv("LUNA_NO_PREWARM"):
|
| 139 |
return
|
| 140 |
threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
# and loudly rather than be hidden behind three slow attempts.
|
| 36 |
_RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3"))
|
| 37 |
_RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5"))
|
| 38 |
+
# Ceiling on how long a provider may put us to sleep. Retry-After is the server's
|
| 39 |
+
# number, and a build has eight minutes in total: a 429 answered with "come back in
|
| 40 |
+
# five minutes" would have parked all eight MAP workers past the deadline and the
|
| 41 |
+
# whole task would have timed out waiting politely. Beyond this we stop waiting and
|
| 42 |
+
# spend the remaining attempt, or give up and let the file degrade.
|
| 43 |
+
_RETRY_MAX_WAIT = float(os.getenv("LUNA_RETRY_MAX_WAIT", "20"))
|
| 44 |
_PERMANENT = ("not supported by any provider", "invalid_request_error", "authorization",
|
| 45 |
"unauthorized", "forbidden", "not found", "does not exist")
|
| 46 |
|
|
|
|
| 80 |
raise
|
| 81 |
if attempt == _RETRY_ATTEMPTS:
|
| 82 |
break
|
| 83 |
+
asked = _retry_after(e)
|
| 84 |
+
wait = asked if asked is not None else (_RETRY_BASE ** attempt) * (0.7 + random.random() * 0.6)
|
| 85 |
+
wait = max(0.0, min(wait, _RETRY_MAX_WAIT))
|
| 86 |
+
capped = " (capped)" if asked is not None and asked > _RETRY_MAX_WAIT else ""
|
| 87 |
print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed "
|
| 88 |
+
f"({type(e).__name__}), retrying in {wait:.1f}s{capped}")
|
| 89 |
time.sleep(wait)
|
| 90 |
raise last
|
| 91 |
|
|
|
|
| 147 |
if os.getenv("LUNA_NO_PREWARM"):
|
| 148 |
return
|
| 149 |
threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start()
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def check_models_async(names) -> None:
|
| 153 |
+
"""Run the probe off the startup path.
|
| 154 |
+
|
| 155 |
+
It used to run inline at import, three round trips before Gradio began serving,
|
| 156 |
+
each able to sit on LLM_TIMEOUT. One unresponsive provider could therefore hold
|
| 157 |
+
the Space back by nine minutes, on a platform that wakes it from sleep for a task
|
| 158 |
+
already capped at eight. The answer is only ever a log line, so nothing waits on
|
| 159 |
+
it."""
|
| 160 |
+
threading.Thread(target=check_models, args=(list(names),),
|
| 161 |
+
name="model-probe", daemon=True).start()
|
|
@@ -457,3 +457,83 @@ def test_past_a_minute_it_becomes_a_clock():
|
|
| 457 |
def test_a_nonsense_time_does_not_raise():
|
| 458 |
assert _stamp(-5) == "0s"
|
| 459 |
assert _stamp(None) == "0s"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 457 |
def test_a_nonsense_time_does_not_raise():
|
| 458 |
assert _stamp(-5) == "0s"
|
| 459 |
assert _stamp(None) == "0s"
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
# --- how long a provider may put us to sleep -----------------------------------
|
| 463 |
+
|
| 464 |
+
class _Resp:
|
| 465 |
+
def __init__(self, status=None, headers=None):
|
| 466 |
+
self.status_code = status
|
| 467 |
+
self.headers = headers or {}
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
class _Err(Exception):
|
| 471 |
+
def __init__(self, msg="rate limited", status=429, retry_after=None):
|
| 472 |
+
super().__init__(msg)
|
| 473 |
+
self.response = _Resp(status, {"Retry-After": str(retry_after)} if retry_after else {})
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
def _run_chat(errors, monkey_sleep):
|
| 477 |
+
"""Drive models.chat with a client that raises the given errors in order."""
|
| 478 |
+
import models
|
| 479 |
+
calls = {"n": 0}
|
| 480 |
+
|
| 481 |
+
def fake(**kw):
|
| 482 |
+
i = calls["n"]; calls["n"] += 1
|
| 483 |
+
if i < len(errors):
|
| 484 |
+
raise errors[i]
|
| 485 |
+
return "ok"
|
| 486 |
+
|
| 487 |
+
old_c, old_s = models.llm_client.chat_completion, models.time.sleep
|
| 488 |
+
models.llm_client.chat_completion = fake
|
| 489 |
+
models.time.sleep = monkey_sleep
|
| 490 |
+
try:
|
| 491 |
+
return models.chat(model="x", messages=[]), calls["n"]
|
| 492 |
+
finally:
|
| 493 |
+
models.llm_client.chat_completion = old_c
|
| 494 |
+
models.time.sleep = old_s
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def test_a_huge_retry_after_is_capped():
|
| 498 |
+
"""A 429 answered "come back in five minutes" used to park a worker past the
|
| 499 |
+
task's own eight-minute deadline."""
|
| 500 |
+
import models
|
| 501 |
+
waits = []
|
| 502 |
+
out, n = _run_chat([_Err(retry_after=300)], waits.append)
|
| 503 |
+
assert out == "ok" and n == 2
|
| 504 |
+
assert waits and waits[0] <= models._RETRY_MAX_WAIT
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
def test_a_short_retry_after_is_honoured():
|
| 508 |
+
waits = []
|
| 509 |
+
_run_chat([_Err(retry_after=2)], waits.append)
|
| 510 |
+
assert waits == [2.0]
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def test_backoff_without_a_header_stays_in_its_band():
|
| 514 |
+
"""Jitter is deliberate - eight MAP workers must not all come back at once - so
|
| 515 |
+
consecutive waits are not strictly ordered. Each one stays in its own band."""
|
| 516 |
+
import models
|
| 517 |
+
waits = []
|
| 518 |
+
_run_chat([_Err(), _Err()], waits.append)
|
| 519 |
+
assert len(waits) == 2
|
| 520 |
+
base = models._RETRY_BASE
|
| 521 |
+
assert base * 0.7 <= waits[0] <= base * 1.3
|
| 522 |
+
assert base ** 2 * 0.7 <= waits[1] <= base ** 2 * 1.3
|
| 523 |
+
assert all(w <= models._RETRY_MAX_WAIT for w in waits)
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
def test_a_permanent_error_is_not_retried():
|
| 527 |
+
"""A revoked token must fail fast, not hide behind three slow attempts."""
|
| 528 |
+
import models, pytest as _pt
|
| 529 |
+
waits = []
|
| 530 |
+
with _pt.raises(Exception):
|
| 531 |
+
_run_chat([_Err("unauthorized", status=401)] * 3, waits.append)
|
| 532 |
+
assert waits == []
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
def test_transient_errors_are_classified_as_such():
|
| 536 |
+
import models
|
| 537 |
+
assert models._is_permanent(_Err("rate limited", status=429)) is False
|
| 538 |
+
assert models._is_permanent(_Err("service unavailable", status=503)) is False
|
| 539 |
+
assert models._is_permanent(_Err("unauthorized", status=401)) is True
|