File size: 6,208 Bytes
107a904
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
"""The NVIDIA fallback must survive a retired model id.

Why this matters: a hardcoded model id is a silent-failure waiting to happen.
When a provider retires a model, the call fails with a 404 that looks exactly
like an expired key from the outside — the fallback returns None, the chain
moves on, and nobody knows the provider stopped working until a student sees
"the AI service is unavailable".

The dangerous half of the fix is the retry. Walking a candidate list on an
AUTH failure would multiply latency by five on a request that is already the
last thing standing between a student and an error message. So these tests
pin both directions: switch on model errors, never on auth or quota errors.
"""
import pytest

from nvidia_fallback import NvidiaFallback, MODEL_CANDIDATES, _is_model_error


class _Err(Exception):
    def __init__(self, message, status=None):
        super().__init__(message)
        self.status_code = status


class _Completions:
    """Accepts exactly one model id; 404s for everything else."""

    def __init__(self, working_model, attempts):
        self.working_model = working_model
        self.attempts = attempts

    def create(self, model, **kwargs):
        self.attempts.append(model)
        if model != self.working_model:
            raise _Err("Model not found", 404)

        message = type("M", (), {"content": f"answer from {model}"})()
        choice = type("C", (), {"message": message})()
        return type("R", (), {"choices": [choice]})()


def _fallback(working_model, attempts, pinned=False):
    n = NvidiaFallback.__new__(NvidiaFallback)
    n.api_key = "test-key"
    n.enabled = True
    n._model_pinned = pinned
    n._exhausted = set()
    n.model = MODEL_CANDIDATES[0]
    completions = _Completions(working_model, attempts)
    n.client = type("Client", (), {
        "chat": type("Chat", (), {"completions": completions})()
    })()
    return n


# ---------------------------------------------------------------- classifier

@pytest.mark.parametrize("exc,expected", [
    (_Err("Model not found", 404), True),
    (_Err("unknown model xyz"), True),
    (_Err("The model is currently unavailable"), True),
    (_Err("Unauthorized", 401), False),
    (_Err("Invalid API key provided"), False),
    (_Err("Rate limit exceeded", 429), False),
    (_Err("Forbidden", 403), False),
    (_Err("Connection timed out"), False),
])
def test_only_model_errors_are_retryable(exc, expected):
    assert _is_model_error(exc) is expected


def test_quota_error_mentioning_model_is_not_retryable():
    """The word "model" appears in plenty of non-model errors.

    "quota exceeded for this model" must not trigger a walk of the candidate
    list — every candidate shares the same quota, so retrying is pure latency.
    """
    assert _is_model_error(_Err("quota exceeded for this model")) is False


@pytest.mark.parametrize("message,status", [
    ("Too many requests for model meta/llama-3.3-70b-instruct", 429),
    ("Access denied for model mistralai/mistral-medium-3.5-128b", 403),
    ("No access to this model", 401),
])
def test_status_code_beats_model_wording(message, status):
    """The case the status-code check exists for.

    Providers name the model in rate-limit and permission errors. Those
    messages contain "model", so the text heuristic alone classifies them as
    retryable and the client walks every candidate on every rate-limited
    request — turning throttling into five times the throttling.

    Verified by mutation: disabling the status-code guard flips these to True
    while every other test in this file still passes, which is precisely why
    they are here.
    """
    assert _is_model_error(_Err(message, status)) is False


# ---------------------------------------------------------------- switching

def test_switches_to_a_working_model_when_the_default_is_retired():
    attempts = []
    target = MODEL_CANDIDATES[2]
    nvidia = _fallback(target, attempts)

    answer = nvidia.generate_response("When is graduation?", "", [])

    assert answer == f"answer from {target}"
    assert nvidia.model == target, "should adopt the model that worked"
    assert attempts[0] == MODEL_CANDIDATES[0], "should try the default first"


def test_adopted_model_is_reused_without_re_probing():
    """The cost control. Without this, every question re-walks the dead
    models and pays their latency again."""
    attempts = []
    target = MODEL_CANDIDATES[2]
    nvidia = _fallback(target, attempts)

    nvidia.generate_response("first", "", [])
    first_call_count = len(attempts)
    attempts.clear()

    nvidia.generate_response("second", "", [])

    assert attempts == [target], "second call must go straight to the adopted model"
    assert first_call_count > 1, "sanity: the first call did have to search"


def test_auth_failure_does_not_walk_the_candidate_list():
    """The expensive mistake. An invalid key fails identically for every
    model, so trying five of them just delays the inevitable."""
    attempts = []
    nvidia = _fallback("nothing-works", attempts)

    def always_unauthorized(model, **kwargs):
        attempts.append(model)
        raise _Err("Unauthorized", 401)

    nvidia.client.chat.completions.create = always_unauthorized

    assert nvidia.generate_response("q", "", []) is None
    assert len(attempts) == 1, (
        f"an auth failure must stop after one attempt, tried {len(attempts)}"
    )


def test_pinned_model_is_never_swapped():
    """If an operator sets NVIDIA_MODEL explicitly, silently using a different
    model would hide their misconfiguration rather than surface it."""
    attempts = []
    nvidia = _fallback(MODEL_CANDIDATES[2], attempts, pinned=True)

    assert nvidia.generate_response("q", "", []) is None
    assert attempts == [MODEL_CANDIDATES[0]], "pinned model must be the only attempt"


def test_returns_none_when_no_candidate_works():
    """Exhausting the list is a normal outcome — the chain moves to the next
    provider. It must not raise."""
    attempts = []
    nvidia = _fallback("nothing-works", attempts)

    assert nvidia.generate_response("q", "", []) is None
    assert len(attempts) == len(MODEL_CANDIDATES)