Spaces:
Running
Running
Download tests/application/test_execution.py from dherain24/claude-proxy: direct link, hf CLI and curl.
- Browser
- Download file 40.1 kB
-
https://huggingface.co/spaces/dherain24/claude-proxy/resolve/main/tests/application/test_execution.py
- Command line
-
hf download hf://spaces/dherain24/claude-proxy/tests/application/test_execution.py
-
curl -L -o test_execution.py https://huggingface.co/spaces/dherain24/claude-proxy/resolve/main/tests/application/test_execution.py
40.1 kB
"""Application-owned provider execution contracts."""
import asyncio
from collections.abc import AsyncIterator, Mapping
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from free_claude_code.application.errors import InvalidRequestError
from free_claude_code.application.execution import ProviderExecutor
from free_claude_code.application.model_metadata import ProviderModelInfo
from free_claude_code.application.routing import (
ProviderModelCandidate,
ProviderModelTarget,
ResolvedModelRoute,
RoutedMessagesRequest,
RoutedResponsesRequest,
)
from free_claude_code.config.reasoning import ReasoningPreference
from free_claude_code.core.anthropic.models import Message, MessagesRequest
from free_claude_code.core.anthropic.passthrough import NativeMessagesRequest
from free_claude_code.core.async_iterators import AsyncCloseable
from free_claude_code.core.chat_completions.request import ChatRequest
from free_claude_code.core.failures import ExecutionFailure, FailureKind
from free_claude_code.core.gemini import GeminiRequest
from free_claude_code.core.openai_responses import OpenAIResponsesRequest
from free_claude_code.core.reasoning import ReasoningCapability, ReasoningPolicy
from free_claude_code.core.stream_delivery import current_stream_delivery
from free_claude_code.core.stream_output import StreamOutput
from free_claude_code.core.stream_recovery import ContinuationSeed
class FakeProvider:
def __init__(self) -> None:
self.stream_calls: list[dict[str, object]] = []
self.stream_close_calls = 0
def stream_chat(
self,
request: ChatRequest,
*,
input_tokens: int,
request_id: str,
response_model: str,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Messages test provider received a Chat request")
def stream_gemini(
self,
request: GeminiRequest,
*,
input_tokens: int,
request_id: str,
response_model: str,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Messages test provider received a Gemini request")
def stream_native_messages(
self,
request: NativeMessagesRequest,
*,
request_id: str,
response_model: str,
request_headers: Mapping[str, str] | None = None,
reasoning_override: ReasoningPolicy | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Compatibility test provider received a native request")
async def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
request_id: str | None = None,
response_model: str | None = None,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
self._record_stream_call(
request,
input_tokens=input_tokens,
request_id=request_id,
response_model=response_model,
reasoning=reasoning,
)
try:
yield StreamOutput(("event: message_stop\ndata: {}\n\n",))
finally:
self.stream_close_calls += 1
async def stream_responses(
self,
request: OpenAIResponsesRequest,
*,
input_tokens: int,
request_id: str,
response_model: str,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Messages test provider received a Responses request")
yield StreamOutput(("",))
def _record_stream_call(
self,
request: MessagesRequest,
*,
input_tokens: int,
request_id: str | None,
response_model: str | None,
reasoning: ReasoningPolicy,
) -> None:
self.stream_calls.append(
{
"request": request,
"input_tokens": input_tokens,
"request_id": request_id,
"response_model": response_model,
"reasoning": reasoning,
}
)
class ResponsesFakeProvider:
def __init__(self) -> None:
self.stream_calls: list[dict[str, object]] = []
self.stream_close_calls = 0
async def stream_messages(
self,
request: MessagesRequest,
*,
input_tokens: int,
request_id: str,
response_model: str,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Responses test provider received a Messages request")
yield StreamOutput(("",))
async def stream_responses(
self,
request: OpenAIResponsesRequest,
*,
input_tokens: int,
request_id: str,
response_model: str,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
self.stream_calls.append(
{
"request": request,
"input_tokens": input_tokens,
"request_id": request_id,
"response_model": response_model,
"reasoning": reasoning,
}
)
try:
yield StreamOutput(("event: response.completed\ndata: {}\n\n",))
finally:
self.stream_close_calls += 1
class WaitStep:
def __init__(self) -> None:
self.started = asyncio.Event()
self.release = asyncio.Event()
class EmptyForever:
pass
EMPTY_FOREVER = EmptyForever()
class DelayStep:
def __init__(self, seconds: float) -> None:
self.seconds = seconds
type StreamStep = str | WaitStep | DelayStep | EmptyForever | BaseException
class ControlledProvider(FakeProvider):
def __init__(self, steps: list[StreamStep]) -> None:
super().__init__()
self._steps = steps
async def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
request_id: str | None = None,
response_model: str | None = None,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
self._record_stream_call(
request,
input_tokens=input_tokens,
request_id=request_id,
response_model=response_model,
reasoning=reasoning,
)
try:
for step in self._steps:
if isinstance(step, str):
yield StreamOutput((step,))
elif isinstance(step, WaitStep):
step.started.set()
await step.release.wait()
elif isinstance(step, DelayStep):
await asyncio.sleep(step.seconds)
elif isinstance(step, EmptyForever):
while True:
yield StreamOutput(("",))
else:
raise step
finally:
self.stream_close_calls += 1
class FailingValidationProvider(FakeProvider):
def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
reasoning: ReasoningPolicy,
model_info: ProviderModelInfo | None = None,
**kwargs: object,
) -> AsyncIterator[StreamOutput]:
raise ValueError("invalid provider request")
class ApplicationErrorValidationProvider(FakeProvider):
def __init__(self, error: InvalidRequestError) -> None:
super().__init__()
self._error = error
def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
reasoning: ReasoningPolicy,
model_info: ProviderModelInfo | None = None,
**kwargs: object,
) -> AsyncIterator[StreamOutput]:
raise self._error
class FailingStreamConstructionProvider(FakeProvider):
def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
request_id: str | None = None,
response_model: str | None = None,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise RuntimeError("stream construction failed")
class ExecutionFailureStreamConstructionProvider(FakeProvider):
def __init__(self, failure: ExecutionFailure) -> None:
super().__init__()
self._failure = failure
def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
request_id: str | None = None,
response_model: str | None = None,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
del request, input_tokens, request_id, response_model, reasoning
raise self._failure
class CloseControlledProvider(FakeProvider):
def __init__(
self,
failure: ExecutionFailure,
*,
close_wait: WaitStep | None = None,
close_error: Exception | None = None,
) -> None:
super().__init__()
self._failure = failure
self._close_wait = close_wait
self._close_error = close_error
self._read = False
def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
request_id: str | None = None,
response_model: str | None = None,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
self._record_stream_call(
request,
input_tokens=input_tokens,
request_id=request_id,
response_model=response_model,
reasoning=reasoning,
)
return self
def __aiter__(self) -> AsyncIterator[StreamOutput]:
return self
async def __anext__(self) -> StreamOutput:
if self._read:
raise StopAsyncIteration
self._read = True
raise self._failure
async def aclose(self) -> None:
self.stream_close_calls += 1
if self._close_wait is not None:
self._close_wait.started.set()
await self._close_wait.release.wait()
if self._close_error is not None:
raise self._close_error
def _target(provider_id: str, provider_model: str) -> ProviderModelTarget:
return ProviderModelTarget(
provider_id=provider_id,
provider_model=provider_model,
provider_model_ref=f"{provider_id}/{provider_model}",
)
def _routed_request(
*fallbacks: ProviderModelTarget,
) -> RoutedMessagesRequest:
request = MessagesRequest(
model="provider-model",
messages=[Message(role="user", content="hello")],
)
return RoutedMessagesRequest(
request=request,
resolved=ResolvedModelRoute(
original_model="gateway-model",
primary=_target("provider", "provider-model"),
fallbacks=tuple(ProviderModelCandidate(target) for target in fallbacks),
reasoning_preference=ReasoningPreference.CLIENT,
),
reasoning=ReasoningPolicy.on(),
)
@pytest.mark.asyncio
@pytest.mark.parametrize("ingress", ["messages", "responses"])
@pytest.mark.parametrize("unknown_fallback", [False, True])
async def test_fallback_receives_its_own_cached_capability_in_stream(
ingress, unknown_fallback
):
seen = []
resolved = set()
class MetadataProvider(ControlledProvider):
async def stream_messages(self, request, input_tokens=0, **kwargs):
seen.append(("stream", request.model, kwargs["model_info"]))
async for event in super().stream_messages(request, input_tokens, **kwargs):
yield event
stream_responses = stream_messages
primary_info = ProviderModelInfo(
"provider/provider-model", reasoning_capability=ReasoningCapability.NONE
)
fallback_info = (
ProviderModelInfo(
"fallback/fallback-model", reasoning_capability=ReasoningCapability.REQUIRED
)
if not unknown_fallback
else None
)
primary = MetadataProvider(
[ExecutionFailure(FailureKind.UNAVAILABLE, 503, "unavailable", True)]
)
fallback = MetadataProvider(["verdict"])
providers = {"provider": primary, "fallback": fallback}
async def resolve(name, _api_key_number):
resolved.add(name)
return providers[name]
def lookup(provider_id, model_id):
assert provider_id in resolved
return {
"provider/provider-model": primary_info,
"fallback/fallback-model": fallback_info,
}.get(f"{provider_id}/{model_id}")
executor = ProviderExecutor(
resolve,
model_info_lookup=lookup,
)
route = _routed_request if ingress == "messages" else _routed_responses_request
output = [
event
async for event in getattr(executor, f"stream_{ingress}")(
route(_target("fallback", "fallback-model")),
raw_log_payload=dict,
request_id="capability-fallback",
)
]
assert output == [StreamOutput(("verdict",))]
assert seen == [
("stream", "provider-model", primary_info),
("stream", "fallback-model", fallback_info),
]
def _routed_responses_request(
*fallbacks: ProviderModelTarget,
) -> RoutedResponsesRequest:
request = OpenAIResponsesRequest(
model="provider-model",
input=[{"role": "user", "content": "hello"}],
reasoning={"effort": "high"},
)
return RoutedResponsesRequest(
request=request,
resolved=ResolvedModelRoute(
original_model="gateway-model",
primary=_target("provider", "provider-model"),
fallbacks=tuple(ProviderModelCandidate(target) for target in fallbacks),
reasoning_preference=ReasoningPreference.CLIENT,
),
reasoning=ReasoningPolicy.on(),
)
def _executor_stream(
provider: FakeProvider,
*,
request_id: str,
) -> AsyncIterator[StreamOutput]:
executor = ProviderExecutor(
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
token_counter=lambda _messages, _system, _tools: 17,
)
return executor.stream_messages(
_routed_request(),
raw_log_payload=dict,
request_id=request_id,
)
@pytest.mark.asyncio
async def test_executor_routes_native_responses_without_messages_conversion() -> None:
provider = ResponsesFakeProvider()
routed = _routed_responses_request()
request = routed.request
executor = ProviderExecutor(
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
responses_token_counter=lambda _request: 23,
)
stream = executor.stream_responses(
routed,
raw_log_payload=lambda: request.model_dump(mode="json"),
request_id="req_responses_application",
)
assert provider.stream_calls == []
assert [frame async for chunk in stream for frame in chunk.frames] == [
"event: response.completed\ndata: {}\n\n"
]
assert provider.stream_calls == [
{
"request": request,
"input_tokens": 23,
"request_id": "req_responses_application",
"response_model": "gateway-model",
"reasoning": ReasoningPolicy.on(),
}
]
assert provider.stream_close_calls == 1
@pytest.mark.asyncio
async def test_executor_uses_structural_provider_port_and_defers_stream_startup() -> (
None
):
provider = FakeProvider()
routed = _routed_request()
request = routed.request
executor = ProviderExecutor(
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
token_counter=lambda _messages, _system, _tools: 17,
)
stream = executor.stream_messages(
routed,
raw_log_payload=request.model_dump,
request_id="req_application",
)
assert provider.stream_calls == []
assert [frame async for chunk in stream for frame in chunk.frames] == [
"event: message_stop\ndata: {}\n\n"
]
assert provider.stream_calls == [
{
"request": request,
"input_tokens": 17,
"request_id": "req_application",
"response_model": "gateway-model",
"reasoning": ReasoningPolicy.on(),
}
]
assert provider.stream_close_calls == 1
def _execution_failure(
message: str,
*,
retryable: bool = True,
) -> ExecutionFailure:
return ExecutionFailure(
kind=FailureKind.OVERLOADED,
status_code=529,
message=message,
retryable=retryable,
)
@pytest.mark.asyncio
async def test_primary_success_never_resolves_fallback() -> None:
primary = FakeProvider()
fallback = FakeProvider()
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": primary, "fallback": fallback}[provider_id]
executor = ProviderExecutor(resolve)
stream = executor.stream_messages(
_routed_request(_target("fallback", "fallback-model")),
raw_log_payload=dict,
request_id="req_primary_success",
)
assert [frame async for chunk in stream for frame in chunk.frames] == [
"event: message_stop\ndata: {}\n\n"
]
assert resolved_ids == ["provider"]
assert fallback.stream_calls == []
@pytest.mark.asyncio
async def test_retryable_preframe_failure_selects_fallback_after_closing_primary() -> (
None
):
order: list[str] = []
primary = ControlledProvider([_execution_failure("primary overloaded")])
fallback = ControlledProvider(["fallback-frame"])
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
if provider_id == "fallback":
assert primary.stream_close_calls == 1
order.append("fallback-resolved")
return {"provider": primary, "fallback": fallback}[provider_id]
executor = ProviderExecutor(
resolve,
token_counter=lambda _messages, _system, _tools: 17,
generation_id=7,
)
routed = _routed_request(_target("fallback", "fallback-model"))
with patch(
"free_claude_code.application.model_fallback.traces.trace_event"
) as trace_mock:
chunks = [
chunk
async for chunk in executor.stream_messages(
routed,
raw_log_payload=dict,
request_id="req_fallback",
)
]
assert chunks == [StreamOutput(("fallback-frame",))]
assert order == ["fallback-resolved"]
assert primary.stream_close_calls == 1
assert fallback.stream_close_calls == 1
fallback_request = fallback.stream_calls[0]["request"]
assert isinstance(fallback_request, MessagesRequest)
assert fallback_request.model == "fallback-model"
assert fallback.stream_calls[0] == {
"request": fallback_request,
"input_tokens": 17,
"request_id": "req_fallback",
"response_model": "gateway-model",
"reasoning": ReasoningPolicy.on(),
}
events = [call.kwargs for call in trace_mock.call_args_list]
started = next(
event
for event in events
if event.get("event") == "free_claude_code.model_fallback.started"
)
assert started == {
"stage": "execution",
"event": "free_claude_code.model_fallback.started",
"source": "application",
"request_id": "req_fallback",
"wire_api": "messages",
"from_provider_model_ref": "provider/provider-model",
"to_provider_model_ref": "fallback/fallback-model",
"from_api_key_number": 1,
"to_api_key_number": 1,
"candidate_index": 2,
"candidate_count": 2,
"failure_kind": "overloaded",
"status_code": 529,
"provider_retryable": True,
"recovery_action": "restart",
"recovery_reason": "no_candidate_output",
"generation_id": 7,
}
selected = next(
event
for event in events
if event.get("event") == "free_claude_code.model_fallback.selected"
)
assert selected["selected_provider_model_ref"] == "fallback/fallback-model"
assert selected["candidate_index"] == 2
@pytest.mark.asyncio
async def test_fallback_chain_preserves_exact_last_failure() -> None:
first = _execution_failure("first")
second = _execution_failure("second")
providers = {
"provider": ControlledProvider([first]),
"second": ControlledProvider([second]),
}
executor = ProviderExecutor(
AsyncMock(
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
),
)
stream = executor.stream_messages(
_routed_request(_target("second", "model")),
raw_log_payload=dict,
request_id="req_exhausted",
)
with pytest.raises(ExecutionFailure) as exc_info:
await anext(stream)
assert exc_info.value is second
assert all(provider.stream_close_calls == 1 for provider in providers.values())
@pytest.mark.asyncio
async def test_multiple_retryable_failures_walk_fallbacks_in_order() -> None:
providers = {
"provider": ControlledProvider([_execution_failure("primary")]),
"second": ControlledProvider([_execution_failure("second")]),
"third": ControlledProvider(["third-frame"]),
}
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return providers[provider_id]
executor = ProviderExecutor(resolve)
stream = executor.stream_messages(
_routed_request(
_target("second", "second-model"),
_target("third", "third-model"),
),
raw_log_payload=dict,
request_id="req_ordered_fallbacks",
)
assert [frame async for chunk in stream for frame in chunk.frames] == [
"third-frame"
]
assert resolved_ids == ["provider", "second", "third"]
assert all(provider.stream_close_calls == 1 for provider in providers.values())
@pytest.mark.asyncio
async def test_empty_primary_completion_does_not_select_fallback() -> None:
primary = ControlledProvider([])
fallback = FakeProvider()
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": primary, "fallback": fallback}[provider_id]
executor = ProviderExecutor(resolve)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_empty_primary",
)
assert [frame async for chunk in stream for frame in chunk.frames] == []
assert resolved_ids == ["provider"]
assert primary.stream_close_calls == 1
@pytest.mark.asyncio
async def test_unexpected_primary_failure_does_not_select_fallback() -> None:
primary = ControlledProvider([RuntimeError("unexpected")])
fallback = FakeProvider()
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": primary, "fallback": fallback}[provider_id]
executor = ProviderExecutor(resolve)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_unexpected_primary",
)
with pytest.raises(RuntimeError, match="unexpected"):
await anext(stream)
assert resolved_ids == ["provider"]
assert primary.stream_close_calls == 1
@pytest.mark.asyncio
async def test_invalid_none_stream_remains_terminal() -> None:
primary = FakeProvider()
fallback = FakeProvider()
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": primary, "fallback": fallback}[provider_id]
executor = ProviderExecutor(resolve)
with patch.object(primary, "stream_messages", return_value=None):
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_invalid_stream",
)
with pytest.raises(TypeError):
await anext(stream)
assert resolved_ids == ["provider"]
assert fallback.stream_calls == []
@pytest.mark.parametrize("failure_kind", tuple(FailureKind))
@pytest.mark.asyncio
async def test_nonretryable_provider_failure_selects_fallback_before_first_frame(
failure_kind: FailureKind,
) -> None:
primary_failure = ExecutionFailure(
kind=failure_kind,
status_code=500,
message="provider rejected candidate",
retryable=False,
)
primary = ControlledProvider([primary_failure])
fallback = ControlledProvider(["fallback-frame"])
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": primary, "fallback": fallback}[provider_id]
executor = ProviderExecutor(resolve)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_rejected",
)
with patch(
"free_claude_code.application.model_fallback.traces.trace_event"
) as trace_mock:
chunks = [chunk async for chunk in stream]
assert chunks == [StreamOutput(("fallback-frame",))]
assert resolved_ids == ["provider", "fallback"]
fallback_started = next(
call.kwargs
for call in trace_mock.call_args_list
if call.kwargs.get("event") == "free_claude_code.model_fallback.started"
)
assert fallback_started["failure_kind"] == failure_kind.value
assert fallback_started["provider_retryable"] is False
@pytest.mark.asyncio
async def test_primary_canonical_startup_failure_selects_fallback() -> None:
primary_failure = ExecutionFailure(
kind=FailureKind.CONTEXT_WINDOW_EXCEEDED,
status_code=400,
message="primary context exceeded",
retryable=False,
)
primary = ExecutionFailureStreamConstructionProvider(primary_failure)
fallback = ControlledProvider(["fallback-frame"])
providers = {"provider": primary, "fallback": fallback}
executor = ProviderExecutor(
AsyncMock(
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
),
)
stream = executor.stream_messages(
_routed_request(_target("fallback", "fallback-model")),
raw_log_payload=dict,
request_id="req_startup_fallback",
)
assert [frame async for chunk in stream for frame in chunk.frames] == [
"fallback-frame"
]
assert primary.stream_calls == []
fallback_request = fallback.stream_calls[0]["request"]
assert isinstance(fallback_request, MessagesRequest)
assert fallback_request.model == "fallback-model"
@pytest.mark.asyncio
async def test_nonretryable_stream_construction_failure_selects_fallback() -> None:
primary_failure = ExecutionFailure(
kind=FailureKind.PERMISSION,
status_code=403,
message="provider denied candidate",
retryable=False,
)
primary = ExecutionFailureStreamConstructionProvider(primary_failure)
fallback = ControlledProvider(["fallback-frame"])
providers = {"provider": primary, "fallback": fallback}
executor = ProviderExecutor(
AsyncMock(
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
),
)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_construction_failure",
)
assert [frame async for chunk in stream for frame in chunk.frames] == [
"fallback-frame"
]
assert fallback.stream_calls[0]["request_id"] == "req_construction_failure"
@pytest.mark.asyncio
async def test_failure_after_first_frame_never_selects_fallback() -> None:
primary_failure = _execution_failure("after frame")
primary = ControlledProvider(["first-frame", primary_failure])
fallback = FakeProvider()
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": primary, "fallback": fallback}[provider_id]
executor = ProviderExecutor(resolve)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_committed",
)
assert await anext(stream) == StreamOutput(("first-frame",))
state = current_stream_delivery()
assert state is not None
state.release_content()
with pytest.raises(ExecutionFailure) as exc_info:
await anext(stream)
assert exc_info.value is primary_failure
assert resolved_ids == ["provider"]
@pytest.mark.asyncio
async def test_lazy_fallback_startup_application_error_stops_unchanged() -> None:
primary = ControlledProvider([_execution_failure("primary")])
startup_error = InvalidRequestError("fallback is misconfigured")
fallback = ApplicationErrorValidationProvider(startup_error)
providers = {"provider": primary, "fallback": fallback}
executor = ProviderExecutor(
AsyncMock(
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
),
)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_startup",
)
with pytest.raises(InvalidRequestError) as exc_info:
await anext(stream)
assert exc_info.value is startup_error
assert primary.stream_close_calls == 1
assert fallback.stream_calls == []
@pytest.mark.asyncio
async def test_candidate_requests_are_isolated_from_provider_mutation() -> None:
class MutatingProvider(ControlledProvider):
async def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
request_id: str | None = None,
response_model: str | None = None,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
request.messages[0].content = "mutated"
async for chunk in super().stream_messages(
request,
input_tokens,
request_id=request_id,
response_model=response_model,
reasoning=reasoning,
):
yield chunk
primary = MutatingProvider([_execution_failure("primary")])
fallback = FakeProvider()
providers = {"provider": primary, "fallback": fallback}
routed = _routed_request(_target("fallback", "model"))
executor = ProviderExecutor(
AsyncMock(
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
),
)
assert [
chunk
async for chunk in executor.stream_messages(
routed,
raw_log_payload=dict,
request_id="req_isolated",
)
]
fallback_request = fallback.stream_calls[0]["request"]
assert isinstance(fallback_request, MessagesRequest)
assert fallback_request.messages[0].content == "hello"
assert routed.request.messages[0].content == "hello"
@pytest.mark.asyncio
async def test_closing_executor_stream_closes_provider_stream_once() -> None:
provider = FakeProvider()
routed = _routed_request()
executor = ProviderExecutor(
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
token_counter=lambda _messages, _system, _tools: 17,
)
stream = executor.stream_messages(
routed,
raw_log_payload=dict,
request_id="req_early_close",
)
assert await anext(stream) == StreamOutput(("event: message_stop\ndata: {}\n\n",))
assert isinstance(stream, AsyncCloseable)
await stream.aclose()
assert provider.stream_close_calls == 1
@pytest.mark.asyncio
async def test_stream_construction_failure_remains_deferred_to_iteration() -> None:
provider = FailingStreamConstructionProvider()
fallback = FakeProvider()
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": provider, "fallback": fallback}[provider_id]
executor = ProviderExecutor(
resolve,
token_counter=lambda _messages, _system, _tools: 17,
)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_deferred_construction",
)
with pytest.raises(RuntimeError, match="stream construction failed"):
await anext(stream)
assert resolved_ids == ["provider"]
@pytest.mark.asyncio
async def test_executor_validation_is_deferred_until_iteration() -> None:
provider = FailingValidationProvider()
token_counter = MagicMock(return_value=17)
executor = ProviderExecutor(
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
token_counter=token_counter,
)
stream = executor.stream_messages(
_routed_request(),
raw_log_payload=dict,
request_id="req_application",
)
token_counter.assert_called_once()
with pytest.raises(ValueError, match="invalid provider request"):
await anext(stream)
assert provider.stream_calls == []
@pytest.mark.asyncio
async def test_cleanup_warning_names_preserved_provider_failure(caplog) -> None:
failure = _execution_failure("primary overloaded")
provider = CloseControlledProvider(
failure,
close_error=RuntimeError("close failed"),
)
stream = _executor_stream(
provider,
request_id="req_close_failure_trace",
)
with pytest.raises(ExecutionFailure) as exc_info:
await anext(stream)
assert exc_info.value is failure
warnings = [record for record in caplog.records if record.levelname == "WARNING"]
assert len(warnings) == 1
assert warnings[0].extra["event"] == "stream.input.close_failed"
assert warnings[0].extra["owner"] == "model_fallback"
assert warnings[0].extra["source"] == "application"
assert warnings[0].extra["close_exc_type"] == "RuntimeError"
assert warnings[0].extra["preserved_exc_type"] == "ExecutionFailure"
assert str(warnings[0].exc_info[1]) == "close failed"
@pytest.mark.asyncio
async def test_provider_owned_timeout_is_not_reclassified() -> None:
provider = ControlledProvider([TimeoutError("provider-owned timeout")])
stream = _executor_stream(
provider,
request_id="req_provider_timeout",
)
with pytest.raises(TimeoutError, match="provider-owned timeout"):
await anext(stream)
assert provider.stream_close_calls == 1
@pytest.mark.asyncio
async def test_cancelling_progress_wait_remains_cancellation() -> None:
wait = WaitStep()
provider = ControlledProvider([wait])
fallback = FakeProvider()
resolved_ids: list[str] = []
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": provider, "fallback": fallback}[provider_id]
executor = ProviderExecutor(
resolve,
)
stream = executor.stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="req_cancelled_progress",
)
task = asyncio.ensure_future(anext(stream))
await wait.started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert provider.stream_close_calls == 1
assert resolved_ids == ["provider"]
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel", [False, True])
async def test_fallback_waits_for_provider_cleanup_and_obeys_cancellation(cancel):
close_wait = WaitStep()
primary = CloseControlledProvider(
_execution_failure("primary overloaded"), close_wait=close_wait
)
fallback = FakeProvider()
resolve = AsyncMock(side_effect=[primary, fallback])
stream = ProviderExecutor(resolve).stream_messages(
_routed_request(_target("fallback", "model")),
raw_log_payload=dict,
request_id="cleanup-wait",
)
async def read():
return await anext(stream)
async with asyncio.timeout(2):
async with asyncio.TaskGroup() as group:
reading = group.create_task(read())
await close_wait.started.wait()
assert not reading.done()
assert resolve.await_count == 1
assert fallback.stream_calls == []
if cancel:
reading.cancel()
with pytest.raises(asyncio.CancelledError):
await reading
assert resolve.await_count == 1
else:
close_wait.release.set()
assert await reading == StreamOutput(
("event: message_stop\ndata: {}\n\n",)
)
assert resolve.await_count == 2
assert isinstance(stream, AsyncCloseable)
await stream.aclose()
assert primary.stream_close_calls == 1
assert fallback.stream_close_calls == (0 if cancel else 1)
@pytest.mark.asyncio
async def test_empty_chunks_allow_caller_cancellation():
provider = ControlledProvider([EMPTY_FOREVER])
stream = _executor_stream(provider, request_id="empty-stream-cancel")
async with asyncio.timeout(2):
task = asyncio.ensure_future(anext(stream))
await asyncio.sleep(0)
assert not task.done()
assert provider.stream_calls
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert provider.stream_close_calls == 1
"""Application-owned provider execution contracts."""
import asyncio
from collections.abc import AsyncIterator, Mapping
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from free_claude_code.application.errors import InvalidRequestError
from free_claude_code.application.execution import ProviderExecutor
from free_claude_code.application.model_metadata import ProviderModelInfo
from free_claude_code.application.routing import (
ProviderModelCandidate,
ProviderModelTarget,
ResolvedModelRoute,
RoutedMessagesRequest,
RoutedResponsesRequest,
)
from free_claude_code.config.reasoning import ReasoningPreference
from free_claude_code.core.anthropic.models import Message, MessagesRequest
from free_claude_code.core.anthropic.passthrough import NativeMessagesRequest
from free_claude_code.core.async_iterators import AsyncCloseable
from free_claude_code.core.chat_completions.request import ChatRequest
from free_claude_code.core.failures import ExecutionFailure, FailureKind
from free_claude_code.core.gemini import GeminiRequest
from free_claude_code.core.openai_responses import OpenAIResponsesRequest
from free_claude_code.core.reasoning import ReasoningCapability, ReasoningPolicy
from free_claude_code.core.stream_delivery import current_stream_delivery
from free_claude_code.core.stream_output import StreamOutput
from free_claude_code.core.stream_recovery import ContinuationSeed
class FakeProvider:
def __init__(self) -> None:
self.stream_calls: list[dict[str, object]] = []
self.stream_close_calls = 0
def stream_chat(
self,
request: ChatRequest,
*,
input_tokens: int,
request_id: str,
response_model: str,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Messages test provider received a Chat request")
def stream_gemini(
self,
request: GeminiRequest,
*,
input_tokens: int,
request_id: str,
response_model: str,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Messages test provider received a Gemini request")
def stream_native_messages(
self,
request: NativeMessagesRequest,
*,
request_id: str,
response_model: str,
request_headers: Mapping[str, str] | None = None,
reasoning_override: ReasoningPolicy | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
raise AssertionError("Compatibility test provider received a native request")
async def stream_messages(
self,
request: MessagesRequest,
input_tokens: int = 0,
*,
request_id: str | None = None,
response_model: str | None = None,
reasoning: ReasoningPolicy,
request_headers: Mapping[str, str] | None = None,
model_info: ProviderModelInfo | None = None,
continuation: ContinuationSeed | None = None,
) -> AsyncIterator[StreamOutput]:
self._record_stream_call(
request,
input_tokens=input_tokens,
request_id=request_id,
response_model=response_model,
reasoning=reasoning,
)
try:
yield StreamOutput(("event: message_stop\ndata: {}\n\n",))
finally:
self.stream_close_calls += 1