msradam commited on
Commit
3368898
·
verified ·
1 Parent(s): d2b160d

fix: vLLM readiness probe + fast Ollama fallback timeout (app/llm.py)

Browse files
Files changed (1) hide show
  1. app/llm.py +3 -1
app/llm.py CHANGED
@@ -97,7 +97,9 @@ def _build_router() -> Router:
97
  # stream_timeout (per-chunk) stays tight since subsequent tokens are fast.
98
  _vllm_first_token_timeout = int(
99
  os.environ.get("RIPRAP_LITELLM_TIMEOUT_S", "360"))
100
- _ollama_timeout = 240
 
 
101
 
102
  for alias, (vllm_name, ollama_tag) in _LOGICAL.items():
103
  if use_vllm:
 
97
  # stream_timeout (per-chunk) stays tight since subsequent tokens are fast.
98
  _vllm_first_token_timeout = int(
99
  os.environ.get("RIPRAP_LITELLM_TIMEOUT_S", "360"))
100
+ # 5s: fail fast so callers (mellea probe loop) aren't blocked waiting
101
+ # for an Ollama that doesn't exist in the vLLM-primary HF Space.
102
+ _ollama_timeout = 5
103
 
104
  for alias, (vllm_name, ollama_tag) in _LOGICAL.items():
105
  if use_vllm: