Spaces:
Running
Running
File size: 3,981 Bytes
9f6a4bd e99df76 691dc7c 3934d27 2e7b8ca 8e438ab 691dc7c 8e438ab 3934d27 8e438ab 3934d27 8e438ab 3934d27 8e438ab 3934d27 8e438ab 3934d27 8e438ab 3934d27 8e438ab 2e7b8ca 691dc7c 2e7b8ca 691dc7c 3934d27 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | from __future__ import annotations
"""Client-owned lifetime boundary for long-running HTTP requests."""
import asyncio
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from .generation_paths import generation_api
_CLIENT_OWNED_PATHS = frozenset(
{
"/v1/messages",
"/v1/responses",
"/v1/messages/count_tokens",
"/admin/api/code/folder-picker",
}
)
class ClientRequestLifetimeMiddleware:
"""Cancel client-owned operations when their HTTP client disconnects."""
def __init__(self, app: ASGIApp) -> None:
self._app = app
async def __call__(
self,
scope: Scope,
receive: Receive,
send: Send,
) -> None:
if not _owns_client_lifetime(scope):
await self._app(scope, receive, send)
return
messages: asyncio.Queue[Message] = asyncio.Queue(maxsize=1)
async def receive_until_disconnect() -> None:
while True:
message = await receive()
if message["type"] == "http.disconnect":
return
await messages.put(message)
async def receive_for_application() -> Message:
return await messages.get()
async def run_application() -> None:
await self._app(scope, receive_for_application, send)
application_task = asyncio.create_task(
run_application(),
name="fcc-client-application",
)
receive_task = asyncio.create_task(
receive_until_disconnect(),
name="fcc-client-disconnect-receiver",
)
try:
done, _pending = await asyncio.wait(
(application_task, receive_task),
return_when=asyncio.FIRST_COMPLETED,
)
if application_task in done:
try:
application_task.result()
finally:
await _cancel_and_wait(receive_task)
return
receive_outcome = _task_outcome(receive_task)
(application_outcome,) = await _cancel_and_wait(application_task)
if receive_outcome is not None:
raise receive_outcome
if application_outcome is not None and not isinstance(
application_outcome,
asyncio.CancelledError,
):
raise application_outcome
except asyncio.CancelledError:
cleanup_task = asyncio.create_task(
_cancel_and_wait(application_task, receive_task),
name="fcc-client-lifetime-cleanup",
)
await _wait_for_cleanup(cleanup_task)
raise
def _owns_client_lifetime(scope: Scope) -> bool:
path = scope.get("path", "")
return scope["type"] == "http" and (
(
scope.get("method") == "POST"
and (path in _CLIENT_OWNED_PATHS or generation_api(path) is not None)
)
or (
scope.get("method") == "GET" and path in {"/v1/models", "/muse-code/models"}
)
)
def _task_outcome(task: asyncio.Task[None]) -> BaseException | None:
try:
task.result()
except BaseException as exc:
return exc
return None
async def _cancel_and_wait(
*tasks: asyncio.Task[None],
) -> tuple[BaseException | None, ...]:
for task in tasks:
if not task.done():
task.cancel()
outcomes: list[BaseException | None] = []
for task in tasks:
try:
await task
except BaseException as exc:
outcomes.append(exc)
else:
outcomes.append(None)
return tuple(outcomes)
async def _wait_for_cleanup(
task: asyncio.Task[tuple[BaseException | None, ...]],
) -> None:
while not task.done():
try:
await asyncio.shield(task)
except asyncio.CancelledError:
continue
task.result()
|