alu-chatbot / test_nvidia_model_fallback.py
Donne120
Claude Opus 5
Fix generation outage: retired Groq model and anthropic SDK signature
107a904
Raw History Blame Contribute Delete
6.21 kB
"""The NVIDIA fallback must survive a retired model id.
Why this matters: a hardcoded model id is a silent-failure waiting to happen.
When a provider retires a model, the call fails with a 404 that looks exactly
like an expired key from the outside — the fallback returns None, the chain
moves on, and nobody knows the provider stopped working until a student sees
"the AI service is unavailable".
The dangerous half of the fix is the retry. Walking a candidate list on an
AUTH failure would multiply latency by five on a request that is already the
last thing standing between a student and an error message. So these tests
pin both directions: switch on model errors, never on auth or quota errors.
"""
import pytest
from nvidia_fallback import NvidiaFallback, MODEL_CANDIDATES, _is_model_error
class _Err(Exception):
def __init__(self, message, status=None):
super().__init__(message)
self.status_code = status
class _Completions:
"""Accepts exactly one model id; 404s for everything else."""
def __init__(self, working_model, attempts):
self.working_model = working_model
self.attempts = attempts
def create(self, model, **kwargs):
self.attempts.append(model)
if model != self.working_model:
raise _Err("Model not found", 404)
message = type("M", (), {"content": f"answer from {model}"})()
choice = type("C", (), {"message": message})()
return type("R", (), {"choices": [choice]})()
def _fallback(working_model, attempts, pinned=False):
n = NvidiaFallback.__new__(NvidiaFallback)
n.api_key = "test-key"
n.enabled = True
n._model_pinned = pinned
n._exhausted = set()
n.model = MODEL_CANDIDATES[0]
completions = _Completions(working_model, attempts)
n.client = type("Client", (), {
"chat": type("Chat", (), {"completions": completions})()
})()
return n
# ---------------------------------------------------------------- classifier
@pytest.mark.parametrize("exc,expected", [
(_Err("Model not found", 404), True),
(_Err("unknown model xyz"), True),
(_Err("The model is currently unavailable"), True),
(_Err("Unauthorized", 401), False),
(_Err("Invalid API key provided"), False),
(_Err("Rate limit exceeded", 429), False),
(_Err("Forbidden", 403), False),
(_Err("Connection timed out"), False),
])
def test_only_model_errors_are_retryable(exc, expected):
assert _is_model_error(exc) is expected
def test_quota_error_mentioning_model_is_not_retryable():
"""The word "model" appears in plenty of non-model errors.
"quota exceeded for this model" must not trigger a walk of the candidate
list — every candidate shares the same quota, so retrying is pure latency.
"""
assert _is_model_error(_Err("quota exceeded for this model")) is False
@pytest.mark.parametrize("message,status", [
("Too many requests for model meta/llama-3.3-70b-instruct", 429),
("Access denied for model mistralai/mistral-medium-3.5-128b", 403),
("No access to this model", 401),
])
def test_status_code_beats_model_wording(message, status):
"""The case the status-code check exists for.
Providers name the model in rate-limit and permission errors. Those
messages contain "model", so the text heuristic alone classifies them as
retryable and the client walks every candidate on every rate-limited
request — turning throttling into five times the throttling.
Verified by mutation: disabling the status-code guard flips these to True
while every other test in this file still passes, which is precisely why
they are here.
"""
assert _is_model_error(_Err(message, status)) is False
# ---------------------------------------------------------------- switching
def test_switches_to_a_working_model_when_the_default_is_retired():
attempts = []
target = MODEL_CANDIDATES[2]
nvidia = _fallback(target, attempts)
answer = nvidia.generate_response("When is graduation?", "", [])
assert answer == f"answer from {target}"
assert nvidia.model == target, "should adopt the model that worked"
assert attempts[0] == MODEL_CANDIDATES[0], "should try the default first"
def test_adopted_model_is_reused_without_re_probing():
"""The cost control. Without this, every question re-walks the dead
models and pays their latency again."""
attempts = []
target = MODEL_CANDIDATES[2]
nvidia = _fallback(target, attempts)
nvidia.generate_response("first", "", [])
first_call_count = len(attempts)
attempts.clear()
nvidia.generate_response("second", "", [])
assert attempts == [target], "second call must go straight to the adopted model"
assert first_call_count > 1, "sanity: the first call did have to search"
def test_auth_failure_does_not_walk_the_candidate_list():
"""The expensive mistake. An invalid key fails identically for every
model, so trying five of them just delays the inevitable."""
attempts = []
nvidia = _fallback("nothing-works", attempts)
def always_unauthorized(model, **kwargs):
attempts.append(model)
raise _Err("Unauthorized", 401)
nvidia.client.chat.completions.create = always_unauthorized
assert nvidia.generate_response("q", "", []) is None
assert len(attempts) == 1, (
f"an auth failure must stop after one attempt, tried {len(attempts)}"
)
def test_pinned_model_is_never_swapped():
"""If an operator sets NVIDIA_MODEL explicitly, silently using a different
model would hide their misconfiguration rather than surface it."""
attempts = []
nvidia = _fallback(MODEL_CANDIDATES[2], attempts, pinned=True)
assert nvidia.generate_response("q", "", []) is None
assert attempts == [MODEL_CANDIDATES[0]], "pinned model must be the only attempt"
def test_returns_none_when_no_candidate_works():
"""Exhausting the list is a normal outcome — the chain moves to the next
provider. It must not raise."""
attempts = []
nvidia = _fallback("nothing-works", attempts)
assert nvidia.generate_response("q", "", []) is None
assert len(attempts) == len(MODEL_CANDIDATES)