Spaces:
Sleeping
Sleeping
File size: 6,208 Bytes
107a904 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | """The NVIDIA fallback must survive a retired model id.
Why this matters: a hardcoded model id is a silent-failure waiting to happen.
When a provider retires a model, the call fails with a 404 that looks exactly
like an expired key from the outside — the fallback returns None, the chain
moves on, and nobody knows the provider stopped working until a student sees
"the AI service is unavailable".
The dangerous half of the fix is the retry. Walking a candidate list on an
AUTH failure would multiply latency by five on a request that is already the
last thing standing between a student and an error message. So these tests
pin both directions: switch on model errors, never on auth or quota errors.
"""
import pytest
from nvidia_fallback import NvidiaFallback, MODEL_CANDIDATES, _is_model_error
class _Err(Exception):
def __init__(self, message, status=None):
super().__init__(message)
self.status_code = status
class _Completions:
"""Accepts exactly one model id; 404s for everything else."""
def __init__(self, working_model, attempts):
self.working_model = working_model
self.attempts = attempts
def create(self, model, **kwargs):
self.attempts.append(model)
if model != self.working_model:
raise _Err("Model not found", 404)
message = type("M", (), {"content": f"answer from {model}"})()
choice = type("C", (), {"message": message})()
return type("R", (), {"choices": [choice]})()
def _fallback(working_model, attempts, pinned=False):
n = NvidiaFallback.__new__(NvidiaFallback)
n.api_key = "test-key"
n.enabled = True
n._model_pinned = pinned
n._exhausted = set()
n.model = MODEL_CANDIDATES[0]
completions = _Completions(working_model, attempts)
n.client = type("Client", (), {
"chat": type("Chat", (), {"completions": completions})()
})()
return n
# ---------------------------------------------------------------- classifier
@pytest.mark.parametrize("exc,expected", [
(_Err("Model not found", 404), True),
(_Err("unknown model xyz"), True),
(_Err("The model is currently unavailable"), True),
(_Err("Unauthorized", 401), False),
(_Err("Invalid API key provided"), False),
(_Err("Rate limit exceeded", 429), False),
(_Err("Forbidden", 403), False),
(_Err("Connection timed out"), False),
])
def test_only_model_errors_are_retryable(exc, expected):
assert _is_model_error(exc) is expected
def test_quota_error_mentioning_model_is_not_retryable():
"""The word "model" appears in plenty of non-model errors.
"quota exceeded for this model" must not trigger a walk of the candidate
list — every candidate shares the same quota, so retrying is pure latency.
"""
assert _is_model_error(_Err("quota exceeded for this model")) is False
@pytest.mark.parametrize("message,status", [
("Too many requests for model meta/llama-3.3-70b-instruct", 429),
("Access denied for model mistralai/mistral-medium-3.5-128b", 403),
("No access to this model", 401),
])
def test_status_code_beats_model_wording(message, status):
"""The case the status-code check exists for.
Providers name the model in rate-limit and permission errors. Those
messages contain "model", so the text heuristic alone classifies them as
retryable and the client walks every candidate on every rate-limited
request — turning throttling into five times the throttling.
Verified by mutation: disabling the status-code guard flips these to True
while every other test in this file still passes, which is precisely why
they are here.
"""
assert _is_model_error(_Err(message, status)) is False
# ---------------------------------------------------------------- switching
def test_switches_to_a_working_model_when_the_default_is_retired():
attempts = []
target = MODEL_CANDIDATES[2]
nvidia = _fallback(target, attempts)
answer = nvidia.generate_response("When is graduation?", "", [])
assert answer == f"answer from {target}"
assert nvidia.model == target, "should adopt the model that worked"
assert attempts[0] == MODEL_CANDIDATES[0], "should try the default first"
def test_adopted_model_is_reused_without_re_probing():
"""The cost control. Without this, every question re-walks the dead
models and pays their latency again."""
attempts = []
target = MODEL_CANDIDATES[2]
nvidia = _fallback(target, attempts)
nvidia.generate_response("first", "", [])
first_call_count = len(attempts)
attempts.clear()
nvidia.generate_response("second", "", [])
assert attempts == [target], "second call must go straight to the adopted model"
assert first_call_count > 1, "sanity: the first call did have to search"
def test_auth_failure_does_not_walk_the_candidate_list():
"""The expensive mistake. An invalid key fails identically for every
model, so trying five of them just delays the inevitable."""
attempts = []
nvidia = _fallback("nothing-works", attempts)
def always_unauthorized(model, **kwargs):
attempts.append(model)
raise _Err("Unauthorized", 401)
nvidia.client.chat.completions.create = always_unauthorized
assert nvidia.generate_response("q", "", []) is None
assert len(attempts) == 1, (
f"an auth failure must stop after one attempt, tried {len(attempts)}"
)
def test_pinned_model_is_never_swapped():
"""If an operator sets NVIDIA_MODEL explicitly, silently using a different
model would hide their misconfiguration rather than surface it."""
attempts = []
nvidia = _fallback(MODEL_CANDIDATES[2], attempts, pinned=True)
assert nvidia.generate_response("q", "", []) is None
assert attempts == [MODEL_CANDIDATES[0]], "pinned model must be the only attempt"
def test_returns_none_when_no_candidate_works():
"""Exhausting the list is a normal outcome — the chain moves to the next
provider. It must not raise."""
attempts = []
nvidia = _fallback("nothing-works", attempts)
assert nvidia.generate_response("q", "", []) is None
assert len(attempts) == len(MODEL_CANDIDATES)
|