claude-proxy / tests /api /test_midstream_model_fallback.py
dheraingoud's picture
Copilot App
Sync upstream v6.19.35 and preserve custom addons
a9e27c0
Raw History Blame Contribute Delete
22.2 kB
"""Model fallback continues the public response through real provider transports."""
import pytest
from free_claude_code.application.model_fallback import stream_model_candidates
from free_claude_code.application.routing import ProviderModelCandidate
from free_claude_code.core.anthropic.stream_contracts import parse_sse_text
from free_claude_code.core.stream_delivery import current_stream_delivery
from free_claude_code.providers.admission import ProviderAdmissionController
from free_claude_code.providers.open_router import OpenRouterProvider
from free_claude_code.providers.recovery_policy import RecoveryTiming
from tests.api.test_delivered_stream_recovery import (
after_text,
assert_completed,
public_text,
text_events,
)
from tests.api.test_hidden_stream_retries import partial_tool
from tests.api.test_recovery_reasoning import followed_by_text, reasoning_events
from tests.api.test_response_streams import _serve
from tests.api.test_tool_call_buffer import _response
from tests.api.test_tool_call_buffer_transports import _tools
from tests.application.test_execution import _routed_request, _target
from tests.providers.support import (
assistant_history_text,
make_provider_config,
without_empty_reasoning,
)
from tests.providers.test_history_transports import _events_for, _harness
from tests.providers.test_native_tool_arguments import tool_events
async def delivered_candidates(senders, wire, *, tools=None, history=None):
if history is None:
history = [{"role": "user", "content": "greet"}]
async def open_candidate(index, candidate, continuation=None):
return senders[index](
wire,
history,
tools=tools,
model=candidate.target.provider_model,
continuation=continuation,
)
routed = _routed_request(
*(
_target(f"fallback-{index}", f"model-{index}")
for index in range(1, len(senders))
)
)
body = stream_model_candidates(
(ProviderModelCandidate(routed.resolved.primary), *routed.resolved.fallbacks),
get_delivery=current_stream_delivery,
wire_api=wire,
request_id="midstream-fallback",
open_candidate=open_candidate,
)
response = await _response(wire, body)
messages = await _serve(response)
return b"".join(message.get("body", b"") for message in messages).decode()
@pytest.mark.asyncio
@pytest.mark.parametrize("wire", ["messages", "responses"])
@pytest.mark.parametrize(
"case",
["full", "summary", "summary_empty_text", "text_empty_part", "summary_whitespace"],
)
async def test_messages_fallback_accepts_partial_output_slots(wire, case):
prefix = (
"Half"
if case == "text_empty_part"
else " "
if case == "summary_whitespace"
else ""
)
if case == "text_empty_part":
first = text_events("responses", prefix, complete=False)
first.append({**first[2], "content_index": 1})
else:
first = reasoning_events(
field="content" if case == "full" else "summary", closed=False
)
if case in {"summary_empty_text", "summary_whitespace"}:
tail = text_events("responses", prefix, complete=False)[1:]
for event in tail:
event["output_index"] = 1
first.extend(tail)
first = [{**event, "sequence_number": index} for index, event in enumerate(first)]
async with (
_harness("responses", lambda _: (200, first), max_attempts=1) as (
primary,
_,
_,
),
_harness(
"messages",
lambda _: (200, text_events("messages", "Recovered.")),
max_attempts=1,
) as (fallback, bodies, _),
):
raw = await delivered_candidates([primary, fallback], wire)
events = parse_sse_text(raw)
assert len(bodies) == 1
assert_completed(events, wire)
assert public_text(events, wire) == prefix + "Recovered."
for message in bodies[0]["messages"]:
if message.get("role") == "assistant":
assert all(
block.get("text") != ""
for block in message["content"]
if block.get("type") == "text"
)
assert ("Find 17." in str(bodies[0]["messages"])) is (case != "text_empty_part")
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["chat", "messages", "responses"])
@pytest.mark.parametrize("target", ["chat", "messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
async def test_exhausted_model_continues_on_next_candidate(source, target, wire):
async with (
_harness(
source,
lambda bodies: (
200,
text_events(
source, ["Hello ", "there. "][len(bodies) - 1], complete=False
),
),
max_attempts=2,
) as (first, first_bodies, _),
_harness(
target, lambda _: (200, text_events(target, "Hello there. Done."))
) as (second, second_bodies, _),
):
raw = await delivered_candidates([first, second], wire, tools=_tools(wire))
assert len(first_bodies) == 2
assert len(second_bodies) == 1
events = parse_sse_text(raw)
assert_completed(events, wire)
assert public_text(events, wire) == "Hello there. Done."
body = second_bodies[0]
history = without_empty_reasoning(body.get("messages", body.get("input")))
assert history[0]["role"] == "user" and "greet" in str(history[0]["content"])
assert assistant_history_text(history) == "Hello there. "
assert sum(row.get("role") == "user" for row in history) == 2
assert body["tools"]
if wire == "responses":
assert (
"".join(
part.get("text", "")
for item in events[-1].data["response"]["output"]
for part in item.get("content", [])
)
== "Hello there. Done."
)
@pytest.mark.asyncio
@pytest.mark.parametrize("target", ["chat", "messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
async def test_fallback_preserves_readable_reasoning_without_a_heading(target, wire):
interrupted = followed_by_text(reasoning_events(closed=False), "Hello ")
async with (
_harness("responses", lambda _: (200, interrupted), max_attempts=1) as (
first,
_,
_,
),
_harness(target, lambda _: (200, text_events(target, "world."))) as (
second,
bodies,
_,
),
):
raw = await delivered_candidates([first, second], wire)
assert len(bodies) == 1
history = bodies[0].get("messages", bodies[0].get("input"))
content = history[-1]["content"]
instruction = content if isinstance(content, str) else content[0]["text"]
assert "Find 17." not in instruction
assert "Find 17." in str(history[:-1])
events = parse_sse_text(raw)
assert_completed(events, wire)
assert public_text(events, wire) == "Hello world."
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ["chat", "messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
async def test_normal_completion_never_opens_another_candidate(protocol, wire):
async with (
_harness(protocol, lambda _: (200, text_events(protocol, "Done."))) as (
first,
first_bodies,
_,
),
_harness("chat", lambda _: (200, text_events("chat", "Unexpected"))) as (
second,
second_bodies,
_,
),
):
raw = await delivered_candidates([first, second], wire)
assert len(first_bodies) == 1
assert not second_bodies
events = parse_sse_text(raw)
assert_completed(events, wire)
assert public_text(events, wire) == "Done."
@pytest.mark.asyncio
@pytest.mark.parametrize("wire", ["messages", "responses"])
async def test_each_model_receives_the_latest_prefix_and_its_own_budget(wire):
async with (
_harness(
"chat",
lambda _: (200, text_events("chat", "One ", complete=False)),
max_attempts=1,
) as (first, first_bodies, _),
_harness(
"messages",
lambda bodies: (
200,
text_events(
"messages", ["two ", "three "][len(bodies) - 1], complete=False
),
),
max_attempts=2,
) as (second, second_bodies, _),
_harness(
"responses",
lambda _: (200, text_events("responses", "One two three four.")),
max_attempts=1,
) as (third, third_bodies, _),
):
raw = await delivered_candidates([first, second, third], wire)
assert [len(first_bodies), len(second_bodies), len(third_bodies)] == [1, 2, 1]
for body, expected in [
(second_bodies[0], "One "),
(second_bodies[1], "One two "),
(third_bodies[0], "One two three "),
]:
history = without_empty_reasoning(body.get("messages", body.get("input")))
assert sum(row.get("role") == "user" for row in history) == 2
assert assistant_history_text(history) == expected
events = parse_sse_text(raw)
assert_completed(events, wire)
assert public_text(events, wire) == "One two three four."
@pytest.mark.asyncio
@pytest.mark.parametrize("wire", ["messages", "responses"])
@pytest.mark.parametrize(
"status,error",
[
(401, {"type": "authentication_error", "message": "Bad credentials"}),
(
400,
{
"type": "invalid_request_error",
"code": "context_length_exceeded",
"message": "Context full",
},
),
],
)
async def test_permanent_rejection_advances_without_erasing_incoming_prefix(
wire, status, error
):
async with (
_harness(
"chat",
lambda _: (200, text_events("chat", "Hello ", complete=False)),
max_attempts=1,
) as (first, first_bodies, _),
_harness("responses", lambda _: (status, error), max_attempts=5) as (
second,
second_bodies,
_,
),
_harness(
"messages",
lambda _: (200, text_events("messages", "world.")),
max_attempts=1,
) as (third, third_bodies, _),
):
raw = await delivered_candidates([first, second, third], wire)
assert [len(first_bodies), len(second_bodies), len(third_bodies)] == [
1,
2 if status == 400 else 1,
1,
]
assert "Hello " in str(third_bodies[0]["messages"][-2])
assert_completed(parse_sse_text(raw), wire)
assert public_text(parse_sse_text(raw), wire) == "Hello world."
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ["chat", "messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
@pytest.mark.parametrize("visible", [False, True])
async def test_abandoned_calls_never_cross_model_boundary(protocol, wire, visible):
failed_events = partial_tool(protocol)
if visible:
failed_events = after_text(protocol, failed_events)
async with (
_harness(protocol, lambda _: (200, failed_events), max_attempts=1) as (
first,
first_bodies,
_,
),
_harness(
"messages",
lambda _: (200, tool_events("messages", '{"path":"winning"}')),
max_attempts=1,
) as (second, second_bodies, _),
):
raw = await delivered_candidates([first, second], wire, tools=_tools(wire))
assert len(first_bodies) == len(second_bodies) == 1
assert "call_abandoned" not in raw and "abandoned" not in str(second_bodies[0])
assert len(second_bodies[0]["messages"]) == (3 if visible else 1)
events = parse_sse_text(raw)
assert_completed(events, wire)
assert public_text(events, wire) == ("Before. " if visible else "")
if wire == "messages":
assert (
"".join(
event.data.get("delta", {}).get("partial_json", "") for event in events
)
== '{"path":"winning"}'
)
else:
calls = [
item
for item in events[-1].data["response"]["output"]
if item["type"] == "function_call"
]
assert len(calls) == 1 and calls[0]["arguments"] == '{"path":"winning"}'
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ["chat", "messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
async def test_first_fallback_call_rejects_native_state_without_trying_next_model(
protocol, wire
):
async with (
_harness(
"chat",
lambda _: (200, text_events("chat", "Hello ", complete=False)),
max_attempts=1,
) as (first, _, _),
_harness(protocol, lambda _: (200, _events_for(protocol)), max_attempts=3) as (
second,
second_bodies,
_,
),
_harness("chat", lambda _: (200, text_events("chat", "Unexpected"))) as (
third,
third_bodies,
_,
),
):
raw = await delivered_candidates([first, second, third], wire)
assert len(second_bodies) == 1 and not third_bodies
events = parse_sse_text(raw)
assert events[-1].event == ("error" if wire == "messages" else "response.failed")
assert public_text(events, wire) == "Hello "
assert "opaque-original" not in raw
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ["chat", "messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
@pytest.mark.parametrize("text", ["", "Hello "])
async def test_first_fallback_call_cannot_complete_without_new_output(
protocol, wire, text
):
async with (
_harness(
"chat",
lambda _: (200, text_events("chat", "Hello ", complete=False)),
max_attempts=1,
) as (first, _, _),
_harness(
protocol, lambda _: (200, text_events(protocol, text)), max_attempts=3
) as (second, second_bodies, _),
_harness("chat", lambda _: (200, text_events("chat", "Unexpected"))) as (
third,
third_bodies,
_,
),
):
raw = await delivered_candidates([first, second, third], wire)
assert len(second_bodies) == 1 and not third_bodies
events = parse_sse_text(raw)
assert events[-1].event == ("error" if wire == "messages" else "response.failed")
assert public_text(events, wire) == "Hello "
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ["messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
@pytest.mark.parametrize("permanent", [False, True])
async def test_released_calls_handoff_only_for_retryable_failure(
protocol, wire, permanent
):
completed = tool_events(protocol, '{"path":"kept"}')
failed_events = completed[:-2] if protocol == "messages" else completed[:-1]
if permanent:
error = {
"type": "invalid_request_error",
"code": "context_length_exceeded",
"message": "Context full",
}
failed_events.append(
{"type": "error", "error": error}
if protocol == "messages"
else {
**completed[-1],
"type": "response.failed",
"response": {
**completed[-1]["response"],
"status": "failed",
"error": error,
},
}
)
async with (
_harness(protocol, lambda _: (200, failed_events), max_attempts=1) as (
first,
first_bodies,
_,
),
_harness("chat", lambda _: (200, text_events("chat", "Unexpected"))) as (
second,
second_bodies,
_,
),
):
raw = await delivered_candidates([first, second], wire, tools=_tools(wire))
assert len(first_bodies) == 1
# Messages block stop alone has no complete call to hand off.
if protocol == "messages":
assert len(second_bodies) == 1
assert "call_probe" not in raw
else:
assert not second_bodies
events = parse_sse_text(raw)
if permanent and protocol != "messages":
assert events[-1].event == (
"error" if wire == "messages" else "response.failed"
)
else:
assert_completed(events, wire)
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ["chat", "messages", "responses"])
@pytest.mark.parametrize("wire", ["messages", "responses"])
async def test_source_native_state_prevents_model_fallback(protocol, wire):
async with (
_harness(
protocol, lambda _: (200, _events_for(protocol)[:-1]), max_attempts=1
) as (first, first_bodies, _),
_harness("chat", lambda _: (200, text_events("chat", "Unexpected"))) as (
second,
second_bodies,
_,
),
):
raw = await delivered_candidates([first, second], wire)
assert len(first_bodies) == 1 and not second_bodies
assert parse_sse_text(raw)[-1].event == (
"error" if wire == "messages" else "response.failed"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("wire", ["messages", "responses"])
@pytest.mark.parametrize("later_failure", [False, True])
async def test_hidden_hosted_activity_remains_a_model_boundary_after_local_retry(
wire, later_failure
):
hidden = [
*partial_tool("responses"),
{
"type": "response.output_item.added",
"sequence_number": 3,
"output_index": 1,
"item": {
"type": "web_search_call",
"id": "ws_hidden",
"status": "in_progress",
},
},
]
def reply(bodies):
if len(bodies) == 1:
return 200, hidden
if later_failure:
return 401, {
"type": "authentication_error",
"message": "Credentials expired",
}
return 200, text_events("responses", "Done.")
async with (
_harness("responses", reply, max_attempts=2) as (first, first_bodies, _),
_harness("chat", lambda _: (200, text_events("chat", "Unexpected"))) as (
second,
second_bodies,
_,
),
):
raw = await delivered_candidates([first, second], wire, tools=_tools(wire))
assert len(first_bodies) == (1 if wire == "responses" else 2)
assert not second_bodies
events = parse_sse_text(raw)
if later_failure or wire == "responses":
assert events[-1].event == (
"error" if wire == "messages" else "response.failed"
)
else:
assert_completed(events, wire)
@pytest.mark.asyncio
@pytest.mark.parametrize("wire", ["messages", "responses"])
@pytest.mark.parametrize("protocol", ["chat", "messages", "responses"])
@pytest.mark.parametrize("partial", [False, True])
async def test_exhausted_model_allows_another_model_on_same_client(
protocol, wire, partial
):
def reply(bodies):
if bodies[-1]["model"] == "model-1":
return 200, text_events(protocol, "world.")
if partial and len(bodies) == 1:
return 200, text_events(protocol, "Hello ", complete=False)
return 503, {"type": "server_error", "message": "Unavailable"}
async with _harness(protocol, reply, max_attempts=2 if partial else 1) as (
send,
bodies,
provider,
):
provider._admission._timing = RecoveryTiming(
base_delay=60, max_delay=60, jitter=0
)
raw = await delivered_candidates([send, send], wire)
blocked = provider._admission._episodes["provider-model"]
assert blocked.terminal_until is not None
expected_models = ["provider-model"] * (2 if partial else 1) + ["model-1"]
assert [body["model"] for body in bodies] == expected_models
events = parse_sse_text(raw)
assert_completed(events, wire)
assert public_text(events, wire) == ("Hello world." if partial else "world.")
@pytest.mark.asyncio
@pytest.mark.parametrize("wire", ["messages", "responses"])
async def test_shared_cooldown_skips_candidate_with_zero_calls_and_keeps_seed(
monkeypatch, wire
):
shared = ProviderAdmissionController(
provider_name="shared",
rate_limit=1000,
max_attempts=2,
base_delay=60,
max_delay=60,
jitter=0,
)
executions = []
start = shared.start_execution
def capture(**kwargs):
execution = start(**kwargs)
executions.append(execution)
return execution
monkeypatch.setattr(shared, "start_execution", capture)
def reply(bodies):
if len(bodies) == 1:
return 200, text_events("chat", "Hello ", complete=False)
return 503, {"type": "server_error", "message": "Unavailable"}
async with (
_harness(
"chat",
reply,
chat_provider_factory=lambda: OpenRouterProvider(
make_provider_config("fixture", "https://provider.invalid/v1"),
admission=shared,
),
) as (first, shared_bodies, _),
_harness("messages", lambda _: (200, text_events("messages", "world."))) as (
third,
third_bodies,
_,
),
):
def same_model(*args, **kwargs):
return first(*args, **{**kwargs, "model": "provider-model"})
raw = await delivered_candidates([first, same_model, third], wire)
assert [execution.attempts_started for execution in executions] == [2, 0]
assert len(shared_bodies) == 2 and len(third_bodies) == 1
assert "Hello " in str(third_bodies[0]["messages"][-2])
assert_completed(parse_sse_text(raw), wire)
assert public_text(parse_sse_text(raw), wire) == "Hello world."