Malina Claude Opus 5 (1M context) commited on
Commit
30ec081
·
1 Parent(s): e4a00dc

Dostawca nie moze uspic nas na dluzej, niz trwa cale zadanie

Browse files

Dwie rzeczy, ktore przy zlym dniu dostawcy inferencji koncza sie zadaniem
przeterminowanym bez jednej linijki Notatki.

1. Ponawianie honorowalo naglowek Retry-After bez zadnego ograniczenia. To
liczba serwera, a build ma osiem minut w sumie: odpowiedz "wroc za piec
minut" na 429 zaparkowalaby wszystkie osiem watkow MAP za linia konca i
cale zadanie przepadloby, czekajac grzecznie. Czas oczekiwania jest teraz
ograniczony do 20 sekund (LUNA_RETRY_MAX_WAIT), a log mowi, kiedy przyciete.

2. check_models wolalo trzy modele synchronicznie przy imporcie, czyli zanim
Gradio zaczelo obslugiwac. Kazde z tych wywolan moze siedziec na
LLM_TIMEOUT, ktory wynosi 180 s, wiec jeden niereagujacy dostawca mogl
wstrzymac wstanie Space'u o dziewiec minut - na platformie, ktora budzi go
ze snu do zadania i tak ograniczonego do osmiu. Sonda idzie teraz w tlo,
bo jej wynikiem jest wylacznie linijka w logu i nic na nia nie czeka.

Testy na oba, plus na to, ze blad trwaly (uniewazniony token) nie jest
ponawiany ani razu. Przy okazji test na sam backoff musial sprawdzac przedzial,
nie kolejnosc: jitter jest celowy, zeby osiem watkow nie wrocilo naraz.

Testy: 47 -> 52.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Files changed (3) hide show
  1. app.py +2 -2
  2. models.py +23 -2
  3. test_luna.py +80 -0
app.py CHANGED
@@ -9,7 +9,7 @@ import gradio as gr
9
 
10
  from auth import verify_usage_token
11
  from config import MAP_MODEL, REDUCE_MODEL, VLM_MODEL
12
- from models import check_models, prewarm_whisper
13
  from pipeline import run
14
  from synthesis import clarify as _clarify
15
 
@@ -24,7 +24,7 @@ def clarify(prompt, token):
24
  # Say out loud, once, whether the configured models are actually served. A name that
25
  # no provider carries any more fails every call, and every call site degrades quietly,
26
  # so the whole extraction stage can be dead for weeks without a single visible error.
27
- check_models([MAP_MODEL, REDUCE_MODEL, VLM_MODEL])
28
  prewarm_whisper()
29
 
30
 
 
9
 
10
  from auth import verify_usage_token
11
  from config import MAP_MODEL, REDUCE_MODEL, VLM_MODEL
12
+ from models import check_models_async, prewarm_whisper
13
  from pipeline import run
14
  from synthesis import clarify as _clarify
15
 
 
24
  # Say out loud, once, whether the configured models are actually served. A name that
25
  # no provider carries any more fails every call, and every call site degrades quietly,
26
  # so the whole extraction stage can be dead for weeks without a single visible error.
27
+ check_models_async([MAP_MODEL, REDUCE_MODEL, VLM_MODEL])
28
  prewarm_whisper()
29
 
30
 
models.py CHANGED
@@ -35,6 +35,12 @@ llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
35
  # and loudly rather than be hidden behind three slow attempts.
36
  _RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3"))
37
  _RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5"))
 
 
 
 
 
 
38
  _PERMANENT = ("not supported by any provider", "invalid_request_error", "authorization",
39
  "unauthorized", "forbidden", "not found", "does not exist")
40
 
@@ -74,9 +80,12 @@ def chat(**kwargs):
74
  raise
75
  if attempt == _RETRY_ATTEMPTS:
76
  break
77
- wait = _retry_after(e) or (_RETRY_BASE ** attempt) * (0.7 + random.random() * 0.6)
 
 
 
78
  print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed "
79
- f"({type(e).__name__}), retrying in {wait:.1f}s")
80
  time.sleep(wait)
81
  raise last
82
 
@@ -138,3 +147,15 @@ def prewarm_whisper() -> None:
138
  if os.getenv("LUNA_NO_PREWARM"):
139
  return
140
  threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start()
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  # and loudly rather than be hidden behind three slow attempts.
36
  _RETRY_ATTEMPTS = int(os.getenv("LUNA_RETRY_ATTEMPTS", "3"))
37
  _RETRY_BASE = float(os.getenv("LUNA_RETRY_BASE", "1.5"))
38
+ # Ceiling on how long a provider may put us to sleep. Retry-After is the server's
39
+ # number, and a build has eight minutes in total: a 429 answered with "come back in
40
+ # five minutes" would have parked all eight MAP workers past the deadline and the
41
+ # whole task would have timed out waiting politely. Beyond this we stop waiting and
42
+ # spend the remaining attempt, or give up and let the file degrade.
43
+ _RETRY_MAX_WAIT = float(os.getenv("LUNA_RETRY_MAX_WAIT", "20"))
44
  _PERMANENT = ("not supported by any provider", "invalid_request_error", "authorization",
45
  "unauthorized", "forbidden", "not found", "does not exist")
46
 
 
80
  raise
81
  if attempt == _RETRY_ATTEMPTS:
82
  break
83
+ asked = _retry_after(e)
84
+ wait = asked if asked is not None else (_RETRY_BASE ** attempt) * (0.7 + random.random() * 0.6)
85
+ wait = max(0.0, min(wait, _RETRY_MAX_WAIT))
86
+ capped = " (capped)" if asked is not None and asked > _RETRY_MAX_WAIT else ""
87
  print(f"[LUNA] {label}: attempt {attempt}/{_RETRY_ATTEMPTS} failed "
88
+ f"({type(e).__name__}), retrying in {wait:.1f}s{capped}")
89
  time.sleep(wait)
90
  raise last
91
 
 
147
  if os.getenv("LUNA_NO_PREWARM"):
148
  return
149
  threading.Thread(target=get_whisper, name="whisper-prewarm", daemon=True).start()
150
+
151
+
152
+ def check_models_async(names) -> None:
153
+ """Run the probe off the startup path.
154
+
155
+ It used to run inline at import, three round trips before Gradio began serving,
156
+ each able to sit on LLM_TIMEOUT. One unresponsive provider could therefore hold
157
+ the Space back by nine minutes, on a platform that wakes it from sleep for a task
158
+ already capped at eight. The answer is only ever a log line, so nothing waits on
159
+ it."""
160
+ threading.Thread(target=check_models, args=(list(names),),
161
+ name="model-probe", daemon=True).start()
test_luna.py CHANGED
@@ -457,3 +457,83 @@ def test_past_a_minute_it_becomes_a_clock():
457
  def test_a_nonsense_time_does_not_raise():
458
  assert _stamp(-5) == "0s"
459
  assert _stamp(None) == "0s"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457
  def test_a_nonsense_time_does_not_raise():
458
  assert _stamp(-5) == "0s"
459
  assert _stamp(None) == "0s"
460
+
461
+
462
+ # --- how long a provider may put us to sleep -----------------------------------
463
+
464
+ class _Resp:
465
+ def __init__(self, status=None, headers=None):
466
+ self.status_code = status
467
+ self.headers = headers or {}
468
+
469
+
470
+ class _Err(Exception):
471
+ def __init__(self, msg="rate limited", status=429, retry_after=None):
472
+ super().__init__(msg)
473
+ self.response = _Resp(status, {"Retry-After": str(retry_after)} if retry_after else {})
474
+
475
+
476
+ def _run_chat(errors, monkey_sleep):
477
+ """Drive models.chat with a client that raises the given errors in order."""
478
+ import models
479
+ calls = {"n": 0}
480
+
481
+ def fake(**kw):
482
+ i = calls["n"]; calls["n"] += 1
483
+ if i < len(errors):
484
+ raise errors[i]
485
+ return "ok"
486
+
487
+ old_c, old_s = models.llm_client.chat_completion, models.time.sleep
488
+ models.llm_client.chat_completion = fake
489
+ models.time.sleep = monkey_sleep
490
+ try:
491
+ return models.chat(model="x", messages=[]), calls["n"]
492
+ finally:
493
+ models.llm_client.chat_completion = old_c
494
+ models.time.sleep = old_s
495
+
496
+
497
+ def test_a_huge_retry_after_is_capped():
498
+ """A 429 answered "come back in five minutes" used to park a worker past the
499
+ task's own eight-minute deadline."""
500
+ import models
501
+ waits = []
502
+ out, n = _run_chat([_Err(retry_after=300)], waits.append)
503
+ assert out == "ok" and n == 2
504
+ assert waits and waits[0] <= models._RETRY_MAX_WAIT
505
+
506
+
507
+ def test_a_short_retry_after_is_honoured():
508
+ waits = []
509
+ _run_chat([_Err(retry_after=2)], waits.append)
510
+ assert waits == [2.0]
511
+
512
+
513
+ def test_backoff_without_a_header_stays_in_its_band():
514
+ """Jitter is deliberate - eight MAP workers must not all come back at once - so
515
+ consecutive waits are not strictly ordered. Each one stays in its own band."""
516
+ import models
517
+ waits = []
518
+ _run_chat([_Err(), _Err()], waits.append)
519
+ assert len(waits) == 2
520
+ base = models._RETRY_BASE
521
+ assert base * 0.7 <= waits[0] <= base * 1.3
522
+ assert base ** 2 * 0.7 <= waits[1] <= base ** 2 * 1.3
523
+ assert all(w <= models._RETRY_MAX_WAIT for w in waits)
524
+
525
+
526
+ def test_a_permanent_error_is_not_retried():
527
+ """A revoked token must fail fast, not hide behind three slow attempts."""
528
+ import models, pytest as _pt
529
+ waits = []
530
+ with _pt.raises(Exception):
531
+ _run_chat([_Err("unauthorized", status=401)] * 3, waits.append)
532
+ assert waits == []
533
+
534
+
535
+ def test_transient_errors_are_classified_as_such():
536
+ import models
537
+ assert models._is_permanent(_Err("rate limited", status=429)) is False
538
+ assert models._is_permanent(_Err("service unavailable", status=503)) is False
539
+ assert models._is_permanent(_Err("unauthorized", status=401)) is True