Spaces:
Running
Running
deploy: phase 3 BYOK backend (Dockerfile.hf, FastAPI on 7860)
Browse files
inference/cloud_clients.py
CHANGED
|
@@ -54,9 +54,14 @@ _retry_on_connection = retry(
|
|
| 54 |
# exceptions raised during iteration escape the retry wrapper). Streaming must
|
| 55 |
# therefore retry by hand, *before the first token is yielded* — once tokens
|
| 56 |
# have streamed we cannot safely replay a partial response.
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
|
| 62 |
def _retry_after_seconds(header_value: str | None) -> float | None:
|
|
|
|
| 54 |
# exceptions raised during iteration escape the retry wrapper). Streaming must
|
| 55 |
# therefore retry by hand, *before the first token is yielded* — once tokens
|
| 56 |
# have streamed we cannot safely replay a partial response.
|
| 57 |
+
# 4 attempts with a tight cadence (≈1.5 + 2.25 + 3.4 s ≈ 7 s worst-case before
|
| 58 |
+
# the first token) keeps the pre-token wait safely under the Vercel Edge 30 s
|
| 59 |
+
# proxy cut while still spanning a few Groq per-minute retries. Retry-After is
|
| 60 |
+
# honoured but capped at _STREAM_BACKOFF_MAX so one slow hint can't blow the
|
| 61 |
+
# Edge budget.
|
| 62 |
+
_STREAM_MAX_ATTEMPTS = 4
|
| 63 |
+
_STREAM_BACKOFF_MIN = 1.5
|
| 64 |
+
_STREAM_BACKOFF_MAX = 12.0
|
| 65 |
|
| 66 |
|
| 67 |
def _retry_after_seconds(header_value: str | None) -> float | None:
|