fix: vLLM readiness probe + fast Ollama fallback timeout (app/llm.py)
Browse files- app/llm.py +3 -1
app/llm.py
CHANGED
|
@@ -97,7 +97,9 @@ def _build_router() -> Router:
|
|
| 97 |
# stream_timeout (per-chunk) stays tight since subsequent tokens are fast.
|
| 98 |
_vllm_first_token_timeout = int(
|
| 99 |
os.environ.get("RIPRAP_LITELLM_TIMEOUT_S", "360"))
|
| 100 |
-
|
|
|
|
|
|
|
| 101 |
|
| 102 |
for alias, (vllm_name, ollama_tag) in _LOGICAL.items():
|
| 103 |
if use_vllm:
|
|
|
|
| 97 |
# stream_timeout (per-chunk) stays tight since subsequent tokens are fast.
|
| 98 |
_vllm_first_token_timeout = int(
|
| 99 |
os.environ.get("RIPRAP_LITELLM_TIMEOUT_S", "360"))
|
| 100 |
+
# 5s: fail fast so callers (mellea probe loop) aren't blocked waiting
|
| 101 |
+
# for an Ollama that doesn't exist in the vLLM-primary HF Space.
|
| 102 |
+
_ollama_timeout = 5
|
| 103 |
|
| 104 |
for alias, (vllm_name, ollama_tag) in _LOGICAL.items():
|
| 105 |
if use_vllm:
|