Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download api/assistant_stream.py from fsanyoto/runloopable: direct link, hf CLI and curl.
- Browser
- Download file 6.92 kB
-
https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/assistant_stream.py
- Command line
-
hf download hf://spaces/fsanyoto/runloopable/api/assistant_stream.py
-
curl -L -o assistant_stream.py https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/assistant_stream.py
6.92 kB
| """Expose one Query turn as server-sent events with same-turn error recovery.""" | |
| import asyncio | |
| import contextvars | |
| import inspect | |
| import json | |
| import threading | |
| import zlib | |
| import anyio | |
| from fastapi import APIRouter, Body, Depends | |
| from starlette.responses import StreamingResponse | |
| from deps import Session, module_gate, require_session | |
| import provider_limits | |
| import provider_stream | |
| PATH = "/api/v1/query/chat/stream" | |
| router = APIRouter(prefix="/api/v1", dependencies=[Depends(module_gate("analyst"))]) | |
| class StreamGZipMiddleware: | |
| """Compress response chunks in the bounded anyio pool while passing SSE through.""" | |
| def __init__(self, app, **kwargs): | |
| self.app = app | |
| self.minimum_size = kwargs.get("minimum_size", 500) | |
| self.compresslevel = kwargs.get("compresslevel", 9) | |
| async def __call__(self, scope, receive, send): | |
| if scope.get("type") != "http" or scope.get("path") == PATH: | |
| return await self.app(scope, receive, send) | |
| from starlette.datastructures import Headers | |
| from starlette.middleware.gzip import IdentityResponder | |
| if "gzip" in Headers(scope=scope).get("Accept-Encoding", ""): | |
| responder = _OffLoopGZipResponder(self.app, self.minimum_size, self.compresslevel) | |
| else: | |
| responder = IdentityResponder(self.app, self.minimum_size) | |
| await responder(scope, receive, send) | |
| from starlette.middleware.gzip import IdentityResponder | |
| class _OffLoopGZipResponder(IdentityResponder): | |
| """Retain Starlette's header and chunk behavior with compression outside the event loop.""" | |
| content_encoding = "gzip" | |
| def __init__(self, app, minimum_size, compresslevel): | |
| super().__init__(app, minimum_size) | |
| self.compresslevel = compresslevel | |
| self._compressor = None | |
| def _compress(self, body, more): | |
| if self._compressor is None: | |
| self._compressor = zlib.compressobj(self.compresslevel, zlib.DEFLATED, 16 + zlib.MAX_WBITS) | |
| out = self._compressor.compress(body) | |
| return out if more else out + self._compressor.flush() | |
| async def send_with_compression(self, message): | |
| if (message["type"] == "http.response.body" | |
| and not self.content_encoding_set and not self.content_type_is_excluded): | |
| body, more = message.get("body", b""), message.get("more_body", False) | |
| if self.started or more or len(body) >= self.minimum_size: | |
| self._compressed_body = await anyio.to_thread.run_sync( | |
| lambda: self._compress(body, more)) | |
| await super().send_with_compression(message) | |
| # Starlette 1.4 awaits apply_compression and 1.3 calls it, so match the installed base. | |
| if inspect.iscoroutinefunction(IdentityResponder.apply_compression): | |
| async def apply_compression(self, body, *, more_body): | |
| return self._compressed_body | |
| else: | |
| def apply_compression(self, body, *, more_body): | |
| return self._compressed_body | |
| def _event(name, data): | |
| return "event: " + name + "\ndata: " + json.dumps(data, ensure_ascii=False) + "\n\n" | |
| async def events(body, session, *, submit=None): | |
| """Run _submit once and send its result or same-turn recovery through the event queue.""" | |
| import routes_query | |
| submit = submit or routes_query._submit | |
| loop = asyncio.get_running_loop() | |
| queue = asyncio.Queue() | |
| emitted = [] | |
| failed = [] | |
| prefix_sent = False | |
| connected = True | |
| def put(name, data): | |
| if connected: | |
| loop.call_soon_threadsafe(queue.put_nowait, (name, data)) | |
| def hook(name, data): | |
| nonlocal prefix_sent | |
| if name == "stream_failure": | |
| failed.append(data["code"]) | |
| return | |
| if name == "thinking": | |
| handle = (routes_query._AGENT_TURN.get() or {}).get("handle") | |
| if handle is not None: | |
| data = {**data, "turnId": handle.id} | |
| if name == "text_delta": | |
| turn = routes_query._AGENT_TURN.get() or {} | |
| handle = turn.get("handle") | |
| if handle is not None and (handle.stopped or handle.cancel_requested.is_set()): | |
| return | |
| if turn.get("providerPaused") and not prefix_sent and not emitted: | |
| prefix = provider_limits.RESUMED_MESSAGE + "\n\n" | |
| emitted.append(prefix) | |
| put("text_delta", {"text": prefix}) | |
| prefix_sent = True | |
| emitted.append(data["text"]) | |
| put(name, data) | |
| def work(): | |
| token = provider_stream.EVENT_HOOK.set(hook) | |
| try: | |
| result = submit(body, session) | |
| if result.get("status") == "joined": | |
| put("joined", result) | |
| elif result.get("status") == "stopped": | |
| put("error", {"code": "turn_stopped", "result": result, "retry": False}) | |
| else: | |
| content = result["message"]["content"] | |
| text = "".join(emitted) | |
| if failed or (text and content != text): | |
| put("error", {"code": failed[0] if failed else "stream_replaced", | |
| "result": result, "retry": False}) | |
| else: | |
| if content[len(text):]: | |
| put("text_delta", {"text": content[len(text):]}) | |
| put("done", result) | |
| except Exception as exc: | |
| from request_timing import resource_cause | |
| cause = resource_cause(exc) | |
| if cause is not None: | |
| put("error", {"code": "server_busy", "cause": cause, | |
| "message": "The server is busy. Try again shortly.", "retry": False}) | |
| return | |
| detail = getattr(exc, "detail", None) | |
| error = (detail or {}).get("error") if isinstance(detail, dict) else None | |
| put("error", {"code": (error or {}).get("code", "stream_failed"), | |
| "message": (error or {}).get("message", "The turn could not finish."), "retry": False}) | |
| finally: | |
| provider_stream.EVENT_HOOK.reset(token) | |
| put("_end", None) | |
| context = contextvars.copy_context() | |
| worker = threading.Thread(target=context.run, args=(work,), daemon=True, name="assistant-stream") | |
| worker.start() | |
| try: | |
| while True: | |
| name, data = await queue.get() | |
| if name == "_end": | |
| break | |
| yield _event(name, data) | |
| finally: | |
| # Disconnect does not replay or cancel writes in an already admitted turn. | |
| connected = False | |
| def stream_chat(body: dict = Body(default=None), session: Session = Depends(require_session)): | |
| return StreamingResponse(events(body, session), media_type="text/event-stream", | |
| headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) | |