File size: 5,473 Bytes
a9e27c0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
from __future__ import annotations
"""Execute ordered model candidates while preserving safe response continuity."""

import asyncio
import sys
from collections.abc import AsyncGenerator, AsyncIterator, Callable

from free_claude_code.core.failures import ExecutionFailure
from free_claude_code.core.request_outcomes import record_request_route
from free_claude_code.core.stream_delivery import StreamDeliveryState
from free_claude_code.core.stream_output import StreamOutput
from free_claude_code.core.stream_recovery import ContinuationSeed
from free_claude_code.core.trace import close_stream_input

from ..routing import ProviderModelCandidate
from .decision import CandidateFailure, Fail, Fallback, Handoff, next_step
from .traces import trace_fallback_selected, trace_fallback_started
from .types import CandidateStreamOpener, WireApi


async def stream_model_candidates(
    candidates: tuple[ProviderModelCandidate, ...],
    open_candidate: CandidateStreamOpener,
    *,
    get_delivery: Callable[[], StreamDeliveryState | None],
    request_id: str,
    wire_api: WireApi,
    generation_id: int | None = None,
) -> AsyncGenerator[StreamOutput]:
    """Run candidates sequentially, closing each before a safe model transition.

    Delivery is acquired on first iteration, after the response consumer binds it.
    The opener owns protocol-specific preparation; this iterator owns its streams.
    """
    delivery = get_delivery()
    if delivery is None:
        raise RuntimeError("Generation requires a response delivery context")
    continuation: ContinuationSeed | None = None
    for index, candidate in enumerate(candidates):
        if index:
            # A replacement can fail before its provider opens an attempt.
            delivery.begin_attempt()
        target = candidate.target
        record_request_route(target.provider_id, target.provider_model)
        provider_stream: AsyncIterator[StreamOutput] | None = None
        candidate_committed = False
        candidate_failure: ExecutionFailure | None = None
        try:
            try:
                provider_stream = await open_candidate(index, candidate, continuation)
            except ExecutionFailure as failure:
                candidate_failure = failure

            if provider_stream is None and candidate_failure is None:
                raise TypeError("provider stream method must return an async iterator")
            while provider_stream is not None:
                try:
                    chunk = await anext(provider_stream)
                except StopAsyncIteration:
                    break
                except ExecutionFailure as failure:
                    candidate_failure = failure
                    break
                if not any(chunk.frames) and not chunk.completed_calls:
                    await asyncio.sleep(0)
                    continue
                if not candidate_committed:
                    candidate_committed = True
                    if index > 0:
                        trace_fallback_selected(
                            request_id=request_id,
                            generation_id=generation_id,
                            wire_api=wire_api,
                            selected=target,
                            candidate_index=index + 1,
                            candidate_count=len(candidates),
                        )
                yield chunk
        finally:
            if provider_stream is not None:
                active_error = sys.exception()
                preserved_error = active_error or candidate_failure
                await close_stream_input(
                    provider_stream,
                    owner="model_fallback",
                    source="application",
                    preserved_error=preserved_error,
                )

        if candidate_failure is None:
            return
        step = next_step(
            CandidateFailure(
                context=candidate_failure.stream_context,
                retryable=candidate_failure.retryable,
                produced_output=candidate_committed,
            ),
            content_released=delivery.content_released,
            # Reading the prefix snapshots the delivered stream; only a
            # failure with stream evidence needs it.
            prefix=(
                delivery.prefix
                if candidate_failure.stream_context is not None
                else None
            ),
            seed=continuation,
            has_next=index + 1 < len(candidates),
        )
        match step:
            case Fail():
                raise candidate_failure
            case Handoff():
                delivery.begin_continuation(handoff=True)
                return
            case Fallback(seed=seed, action=action, reason=reason):
                continuation = seed
                if continuation is not None:
                    delivery.begin_continuation()
                trace_fallback_started(
                    request_id=request_id,
                    generation_id=generation_id,
                    wire_api=wire_api,
                    failed=target,
                    selected=candidates[index + 1].target,
                    failure=candidate_failure,
                    candidate_index=index + 2,
                    candidate_count=len(candidates),
                    recovery_action=action.value,
                    recovery_reason=reason,
                )