LeomordKaly commited on
Commit
fa8cdfa
·
verified ·
1 Parent(s): 48f9ef6

deploy: phase 3 BYOK backend (Dockerfile.hf, FastAPI on 7860)

Browse files
Files changed (1) hide show
  1. inference/cloud_clients.py +8 -3
inference/cloud_clients.py CHANGED
@@ -54,9 +54,14 @@ _retry_on_connection = retry(
54
  # exceptions raised during iteration escape the retry wrapper). Streaming must
55
  # therefore retry by hand, *before the first token is yielded* — once tokens
56
  # have streamed we cannot safely replay a partial response.
57
- _STREAM_MAX_ATTEMPTS = 3
58
- _STREAM_BACKOFF_MIN = 2.0
59
- _STREAM_BACKOFF_MAX = 20.0
 
 
 
 
 
60
 
61
 
62
  def _retry_after_seconds(header_value: str | None) -> float | None:
 
54
  # exceptions raised during iteration escape the retry wrapper). Streaming must
55
  # therefore retry by hand, *before the first token is yielded* — once tokens
56
  # have streamed we cannot safely replay a partial response.
57
+ # 4 attempts with a tight cadence (≈1.5 + 2.25 + 3.4 s ≈ 7 s worst-case before
58
+ # the first token) keeps the pre-token wait safely under the Vercel Edge 30 s
59
+ # proxy cut while still spanning a few Groq per-minute retries. Retry-After is
60
+ # honoured but capped at _STREAM_BACKOFF_MAX so one slow hint can't blow the
61
+ # Edge budget.
62
+ _STREAM_MAX_ATTEMPTS = 4
63
+ _STREAM_BACKOFF_MIN = 1.5
64
+ _STREAM_BACKOFF_MAX = 12.0
65
 
66
 
67
  def _retry_after_seconds(header_value: str | None) -> float | None: