Spaces:
Running
Running
File size: 5,473 Bytes
a9e27c0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | from __future__ import annotations
"""Execute ordered model candidates while preserving safe response continuity."""
import asyncio
import sys
from collections.abc import AsyncGenerator, AsyncIterator, Callable
from free_claude_code.core.failures import ExecutionFailure
from free_claude_code.core.request_outcomes import record_request_route
from free_claude_code.core.stream_delivery import StreamDeliveryState
from free_claude_code.core.stream_output import StreamOutput
from free_claude_code.core.stream_recovery import ContinuationSeed
from free_claude_code.core.trace import close_stream_input
from ..routing import ProviderModelCandidate
from .decision import CandidateFailure, Fail, Fallback, Handoff, next_step
from .traces import trace_fallback_selected, trace_fallback_started
from .types import CandidateStreamOpener, WireApi
async def stream_model_candidates(
candidates: tuple[ProviderModelCandidate, ...],
open_candidate: CandidateStreamOpener,
*,
get_delivery: Callable[[], StreamDeliveryState | None],
request_id: str,
wire_api: WireApi,
generation_id: int | None = None,
) -> AsyncGenerator[StreamOutput]:
"""Run candidates sequentially, closing each before a safe model transition.
Delivery is acquired on first iteration, after the response consumer binds it.
The opener owns protocol-specific preparation; this iterator owns its streams.
"""
delivery = get_delivery()
if delivery is None:
raise RuntimeError("Generation requires a response delivery context")
continuation: ContinuationSeed | None = None
for index, candidate in enumerate(candidates):
if index:
# A replacement can fail before its provider opens an attempt.
delivery.begin_attempt()
target = candidate.target
record_request_route(target.provider_id, target.provider_model)
provider_stream: AsyncIterator[StreamOutput] | None = None
candidate_committed = False
candidate_failure: ExecutionFailure | None = None
try:
try:
provider_stream = await open_candidate(index, candidate, continuation)
except ExecutionFailure as failure:
candidate_failure = failure
if provider_stream is None and candidate_failure is None:
raise TypeError("provider stream method must return an async iterator")
while provider_stream is not None:
try:
chunk = await anext(provider_stream)
except StopAsyncIteration:
break
except ExecutionFailure as failure:
candidate_failure = failure
break
if not any(chunk.frames) and not chunk.completed_calls:
await asyncio.sleep(0)
continue
if not candidate_committed:
candidate_committed = True
if index > 0:
trace_fallback_selected(
request_id=request_id,
generation_id=generation_id,
wire_api=wire_api,
selected=target,
candidate_index=index + 1,
candidate_count=len(candidates),
)
yield chunk
finally:
if provider_stream is not None:
active_error = sys.exception()
preserved_error = active_error or candidate_failure
await close_stream_input(
provider_stream,
owner="model_fallback",
source="application",
preserved_error=preserved_error,
)
if candidate_failure is None:
return
step = next_step(
CandidateFailure(
context=candidate_failure.stream_context,
retryable=candidate_failure.retryable,
produced_output=candidate_committed,
),
content_released=delivery.content_released,
# Reading the prefix snapshots the delivered stream; only a
# failure with stream evidence needs it.
prefix=(
delivery.prefix
if candidate_failure.stream_context is not None
else None
),
seed=continuation,
has_next=index + 1 < len(candidates),
)
match step:
case Fail():
raise candidate_failure
case Handoff():
delivery.begin_continuation(handoff=True)
return
case Fallback(seed=seed, action=action, reason=reason):
continuation = seed
if continuation is not None:
delivery.begin_continuation()
trace_fallback_started(
request_id=request_id,
generation_id=generation_id,
wire_api=wire_api,
failed=target,
selected=candidates[index + 1].target,
failure=candidate_failure,
candidate_index=index + 2,
candidate_count=len(candidates),
recovery_action=action.value,
recovery_reason=reason,
)
|