File size: 3,981 Bytes
9f6a4bd
e99df76
691dc7c
3934d27
 
 
 
2e7b8ca
 
8e438ab
691dc7c
 
 
 
 
 
8e438ab
3934d27
 
8e438ab
 
3934d27
 
 
 
 
 
 
 
 
 
8e438ab
3934d27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8e438ab
3934d27
 
 
8e438ab
3934d27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8e438ab
3934d27
 
 
 
 
8e438ab
2e7b8ca
691dc7c
2e7b8ca
 
 
 
691dc7c
 
 
3934d27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
from __future__ import annotations
"""Client-owned lifetime boundary for long-running HTTP requests."""

import asyncio

from starlette.types import ASGIApp, Message, Receive, Scope, Send

from .generation_paths import generation_api

_CLIENT_OWNED_PATHS = frozenset(
    {
        "/v1/messages",
        "/v1/responses",
        "/v1/messages/count_tokens",
        "/admin/api/code/folder-picker",
    }
)


class ClientRequestLifetimeMiddleware:
    """Cancel client-owned operations when their HTTP client disconnects."""

    def __init__(self, app: ASGIApp) -> None:
        self._app = app

    async def __call__(
        self,
        scope: Scope,
        receive: Receive,
        send: Send,
    ) -> None:
        if not _owns_client_lifetime(scope):
            await self._app(scope, receive, send)
            return

        messages: asyncio.Queue[Message] = asyncio.Queue(maxsize=1)

        async def receive_until_disconnect() -> None:
            while True:
                message = await receive()
                if message["type"] == "http.disconnect":
                    return
                await messages.put(message)

        async def receive_for_application() -> Message:
            return await messages.get()

        async def run_application() -> None:
            await self._app(scope, receive_for_application, send)

        application_task = asyncio.create_task(
            run_application(),
            name="fcc-client-application",
        )
        receive_task = asyncio.create_task(
            receive_until_disconnect(),
            name="fcc-client-disconnect-receiver",
        )
        try:
            done, _pending = await asyncio.wait(
                (application_task, receive_task),
                return_when=asyncio.FIRST_COMPLETED,
            )
            if application_task in done:
                try:
                    application_task.result()
                finally:
                    await _cancel_and_wait(receive_task)
                return

            receive_outcome = _task_outcome(receive_task)
            (application_outcome,) = await _cancel_and_wait(application_task)
            if receive_outcome is not None:
                raise receive_outcome
            if application_outcome is not None and not isinstance(
                application_outcome,
                asyncio.CancelledError,
            ):
                raise application_outcome
        except asyncio.CancelledError:
            cleanup_task = asyncio.create_task(
                _cancel_and_wait(application_task, receive_task),
                name="fcc-client-lifetime-cleanup",
            )
            await _wait_for_cleanup(cleanup_task)
            raise


def _owns_client_lifetime(scope: Scope) -> bool:
    path = scope.get("path", "")
    return scope["type"] == "http" and (
        (
            scope.get("method") == "POST"
            and (path in _CLIENT_OWNED_PATHS or generation_api(path) is not None)
        )
        or (
            scope.get("method") == "GET" and path in {"/v1/models", "/muse-code/models"}
        )
    )


def _task_outcome(task: asyncio.Task[None]) -> BaseException | None:
    try:
        task.result()
    except BaseException as exc:
        return exc
    return None


async def _cancel_and_wait(
    *tasks: asyncio.Task[None],
) -> tuple[BaseException | None, ...]:
    for task in tasks:
        if not task.done():
            task.cancel()

    outcomes: list[BaseException | None] = []
    for task in tasks:
        try:
            await task
        except BaseException as exc:
            outcomes.append(exc)
        else:
            outcomes.append(None)
    return tuple(outcomes)


async def _wait_for_cleanup(
    task: asyncio.Task[tuple[BaseException | None, ...]],
) -> None:
    while not task.done():
        try:
            await asyncio.shield(task)
        except asyncio.CancelledError:
            continue
    task.result()