File size: 6,916 Bytes
89af3b2
 
 
07a68d0
89af3b2
 
07a68d0
89af3b2
92127d3
 
89af3b2
 
 
 
 
 
 
 
 
 
 
 
92127d3
89af3b2
 
 
92127d3
 
89af3b2
 
92127d3
 
 
 
 
 
 
 
 
 
 
07a68d0
92127d3
 
07a68d0
92127d3
 
07a68d0
 
92127d3
07a68d0
92127d3
07a68d0
92127d3
 
07a68d0
 
 
 
92127d3
 
 
 
 
 
 
 
 
 
07a68d0
 
 
 
 
 
 
89af3b2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92127d3
 
 
 
 
 
89af3b2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
"""Expose one Query turn as server-sent events with same-turn error recovery."""
import asyncio
import contextvars
import inspect
import json
import threading
import zlib

import anyio

from fastapi import APIRouter, Body, Depends
from starlette.responses import StreamingResponse

from deps import Session, module_gate, require_session
import provider_limits
import provider_stream

PATH = "/api/v1/query/chat/stream"
router = APIRouter(prefix="/api/v1", dependencies=[Depends(module_gate("analyst"))])


class StreamGZipMiddleware:
    """Compress response chunks in the bounded anyio pool while passing SSE through."""

    def __init__(self, app, **kwargs):
        self.app = app
        self.minimum_size = kwargs.get("minimum_size", 500)
        self.compresslevel = kwargs.get("compresslevel", 9)

    async def __call__(self, scope, receive, send):
        if scope.get("type") != "http" or scope.get("path") == PATH:
            return await self.app(scope, receive, send)
        from starlette.datastructures import Headers
        from starlette.middleware.gzip import IdentityResponder
        if "gzip" in Headers(scope=scope).get("Accept-Encoding", ""):
            responder = _OffLoopGZipResponder(self.app, self.minimum_size, self.compresslevel)
        else:
            responder = IdentityResponder(self.app, self.minimum_size)
        await responder(scope, receive, send)


from starlette.middleware.gzip import IdentityResponder


class _OffLoopGZipResponder(IdentityResponder):
    """Retain Starlette's header and chunk behavior with compression outside the event loop."""

    content_encoding = "gzip"

    def __init__(self, app, minimum_size, compresslevel):
        super().__init__(app, minimum_size)
        self.compresslevel = compresslevel
        self._compressor = None

    def _compress(self, body, more):
        if self._compressor is None:
            self._compressor = zlib.compressobj(self.compresslevel, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
        out = self._compressor.compress(body)
        return out if more else out + self._compressor.flush()

    async def send_with_compression(self, message):
        if (message["type"] == "http.response.body"
                and not self.content_encoding_set and not self.content_type_is_excluded):
            body, more = message.get("body", b""), message.get("more_body", False)
            if self.started or more or len(body) >= self.minimum_size:
                self._compressed_body = await anyio.to_thread.run_sync(
                    lambda: self._compress(body, more))
        await super().send_with_compression(message)

    # Starlette 1.4 awaits apply_compression and 1.3 calls it, so match the installed base.
    if inspect.iscoroutinefunction(IdentityResponder.apply_compression):
        async def apply_compression(self, body, *, more_body):
            return self._compressed_body
    else:
        def apply_compression(self, body, *, more_body):
            return self._compressed_body


def _event(name, data):
    return "event: " + name + "\ndata: " + json.dumps(data, ensure_ascii=False) + "\n\n"


async def events(body, session, *, submit=None):
    """Run _submit once and send its result or same-turn recovery through the event queue."""
    import routes_query
    submit = submit or routes_query._submit
    loop = asyncio.get_running_loop()
    queue = asyncio.Queue()
    emitted = []
    failed = []
    prefix_sent = False
    connected = True

    def put(name, data):
        if connected:
            loop.call_soon_threadsafe(queue.put_nowait, (name, data))

    def hook(name, data):
        nonlocal prefix_sent
        if name == "stream_failure":
            failed.append(data["code"])
            return
        if name == "thinking":
            handle = (routes_query._AGENT_TURN.get() or {}).get("handle")
            if handle is not None:
                data = {**data, "turnId": handle.id}
        if name == "text_delta":
            turn = routes_query._AGENT_TURN.get() or {}
            handle = turn.get("handle")
            if handle is not None and (handle.stopped or handle.cancel_requested.is_set()):
                return
            if turn.get("providerPaused") and not prefix_sent and not emitted:
                prefix = provider_limits.RESUMED_MESSAGE + "\n\n"
                emitted.append(prefix)
                put("text_delta", {"text": prefix})
                prefix_sent = True
            emitted.append(data["text"])
        put(name, data)

    def work():
        token = provider_stream.EVENT_HOOK.set(hook)
        try:
            result = submit(body, session)
            if result.get("status") == "joined":
                put("joined", result)
            elif result.get("status") == "stopped":
                put("error", {"code": "turn_stopped", "result": result, "retry": False})
            else:
                content = result["message"]["content"]
                text = "".join(emitted)
                if failed or (text and content != text):
                    put("error", {"code": failed[0] if failed else "stream_replaced",
                                  "result": result, "retry": False})
                else:
                    if content[len(text):]:
                        put("text_delta", {"text": content[len(text):]})
                    put("done", result)
        except Exception as exc:
            from request_timing import resource_cause
            cause = resource_cause(exc)
            if cause is not None:
                put("error", {"code": "server_busy", "cause": cause,
                              "message": "The server is busy. Try again shortly.", "retry": False})
                return
            detail = getattr(exc, "detail", None)
            error = (detail or {}).get("error") if isinstance(detail, dict) else None
            put("error", {"code": (error or {}).get("code", "stream_failed"),
                          "message": (error or {}).get("message", "The turn could not finish."), "retry": False})
        finally:
            provider_stream.EVENT_HOOK.reset(token)
            put("_end", None)

    context = contextvars.copy_context()
    worker = threading.Thread(target=context.run, args=(work,), daemon=True, name="assistant-stream")
    worker.start()
    try:
        while True:
            name, data = await queue.get()
            if name == "_end":
                break
            yield _event(name, data)
    finally:
        # Disconnect does not replay or cancel writes in an already admitted turn.
        connected = False


@router.post("/query/chat/stream")
def stream_chat(body: dict = Body(default=None), session: Session = Depends(require_session)):
    return StreamingResponse(events(body, session), media_type="text/event-stream",
                             headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})