"""Translate the router's OpenAI-shaped SSE into the events the UI consumes. The translation is a pure function over decoded frames so it can be tested against recorded fixtures with no network and no credentials, which is what lets the test job gate the deploy without ever holding a token. """ from __future__ import annotations import json from dataclasses import dataclass, field from typing import Iterator @dataclass class Accumulated: """Running totals, so a stream that dies mid-flight still reports what it got.""" content: str = "" reasoning: str = "" stats: dict = field(default_factory=dict) finished: bool = False # Tool calls arrive as fragments keyed by index: the name in one chunk, the # arguments a character at a time across dozens more. They are only useful # once the whole call has landed, so they are assembled here and emitted at # the end rather than forwarded piecemeal. tool_calls: dict = field(default_factory=dict) tools_sent: bool = False # Some providers drop the separate reasoning channel and inline the model's # thinking as ... in the content instead. `thinking` tracks # which side of that tag we are on, and `held` is the part of a tag that # arrived split across two chunks. thinking: bool = False held: str = "" def sse(event: str, data: str) -> str: """One SSE frame. Data is split so a newline in the payload cannot end it.""" body = "".join(f"data: {line}\n" for line in data.split("\n")) return f"event: {event}\n{body}\n" def translate(raw_line: str, acc: Accumulated) -> Iterator[str]: """Feed one upstream line; yield zero or more frames for the browser.""" line = raw_line.strip() if not line or not line.startswith("data:"): return payload = line[5:].strip() if payload == "[DONE]": # Anything still held back was the start of a tag that never completed, # so it was ordinary text after all. if acc.held: acc.content += acc.held yield sse("delta", json.dumps({"content": acc.held})) acc.held = "" acc.finished = True yield from _tool_frame(acc) yield sse("stats", json.dumps(acc.stats)) return try: chunk = json.loads(payload) except json.JSONDecodeError: return # Some providers report an error inside a 200 stream rather than as a # status code, which is otherwise indistinguishable from a silent stall. if isinstance(chunk.get("error"), (dict, str)): err = chunk["error"] msg = err.get("message") if isinstance(err, dict) else str(err) yield sse("error", msg or "The provider returned an error.") return choices = chunk.get("choices") or [] if choices: choice = choices[0] delta = choice.get("delta") or {} out: dict = {} if delta.get("content"): body, inline_thought = split_thinking(delta["content"], acc) if body: out["content"] = body acc.content += body if inline_thought: out["reasoning"] = inline_thought acc.reasoning += inline_thought # GLM-5.3 streams thinking on its own channel; some providers spell it # `reasoning` rather than `reasoning_content`. thought = delta.get("reasoning_content") or delta.get("reasoning") if thought: out["reasoning"] = out.get("reasoning", "") + thought acc.reasoning += thought if out: yield sse("delta", json.dumps(out)) _collect_tool_calls(delta.get("tool_calls"), acc) if choice.get("finish_reason"): acc.stats["finishReason"] = choice["finish_reason"] if choice["finish_reason"] == "tool_calls": yield from _tool_frame(acc) if chunk.get("id"): acc.stats["requestId"] = chunk["id"] if chunk.get("model"): acc.stats["model"] = chunk["model"].split("/")[-1] usage = chunk.get("usage") if isinstance(usage, dict): acc.stats.update(_usage(usage)) yield sse("stats", json.dumps(acc.stats)) THINK_OPEN = "" THINK_CLOSE = "" def split_thinking(text: str, acc: Accumulated) -> tuple[str, str]: """Route one content delta into (answer, thinking). A model whose provider drops the reasoning channel writes its thinking into the content as ``…``, and the reader gets the scratchpad and a stray closing tag in the middle of the answer. The tags straddle chunk boundaries constantly -- ```` -- so a trailing fragment that could still become one is held back until the next chunk decides. Stateful across the stream, pure in everything else. """ buf = acc.held + text acc.held = "" answer: list[str] = [] thinking: list[str] = [] while buf: tag = THINK_CLOSE if acc.thinking else THINK_OPEN into = thinking if acc.thinking else answer at = buf.find(tag) if at >= 0: into.append(buf[:at]) buf = buf[at + len(tag):] acc.thinking = not acc.thinking continue keep = _partial_tag(buf, tag) into.append(buf[:len(buf) - keep] if keep else buf) acc.held = buf[len(buf) - keep:] if keep else "" break return "".join(answer), "".join(thinking) def _partial_tag(text: str, tag: str) -> int: """Length of the trailing fragment that could still grow into ``tag``.""" for n in range(min(len(tag) - 1, len(text)), 0, -1): if text.endswith(tag[:n]): return n return 0 def _collect_tool_calls(chunks, acc: Accumulated) -> None: """Merge one chunk's worth of tool-call fragments into the accumulator.""" for tc in chunks or []: if not isinstance(tc, dict): continue # A provider that omits `index` is streaming one call at a time. index = tc.get("index", 0) slot = acc.tool_calls.setdefault(index, {"id": "", "name": "", "arguments": ""}) if tc.get("id"): slot["id"] = str(tc["id"]) fn = tc.get("function") or {} if fn.get("name"): slot["name"] = str(fn["name"]) if fn.get("arguments"): slot["arguments"] += str(fn["arguments"]) def _tool_frame(acc: Accumulated) -> Iterator[str]: """The assembled calls, once. Emitted at `tool_calls` or at [DONE].""" if acc.tools_sent or not acc.tool_calls: return acc.tools_sent = True calls = [] for index in sorted(acc.tool_calls): call = acc.tool_calls[index] # A fragment that never carried a name is an incomplete call; forwarding # it would have the client invoke a tool called "". if not call["name"]: continue # Some providers omit the id entirely. The round trip only needs it to # be unique within the turn, and the model is handed back whatever we # send here, so an index-derived one is as good as theirs. call["id"] = call["id"] or f"call_{index}_{call['name']}" calls.append(call) if calls: yield sse("tools", json.dumps(calls)) def _usage(u: dict) -> dict: """Flatten the usage block, including the fields most clients drop.""" out: dict = {} for src, dst in ( ("prompt_tokens", "promptTokens"), ("completion_tokens", "completionTokens"), ): if u.get(src) is not None: out[dst] = u[src] prompt_detail = u.get("prompt_tokens_details") or {} if prompt_detail.get("cached_tokens") is not None: out["cachedTokens"] = prompt_detail["cached_tokens"] detail = u.get("completion_tokens_details") or {} for src, dst in ( ("reasoning_tokens", "reasoningTokens"), ("accepted_prediction_tokens", "acceptedTokens"), ("rejected_prediction_tokens", "rejectedTokens"), ): if detail.get(src) is not None: out[dst] = detail[src] return out MAX_TOOLS = 32 _ROLES = ("system", "user", "assistant", "tool") def normalise_messages(messages: list) -> list: """Turn the client's messages into the shape the router expects. The browser speaks the same camelCase it uses everywhere else; the OpenAI wire format is snake_case and nests a tool call inside a `function` object. Translating here rather than in the client keeps one JSON shape in the UI and one on the wire, and gives the round trip something to test. An assistant message that asked for tools must keep its `tool_calls`, and every one of them must be answered by a `tool` message: providers reject the pair broken apart. """ out: list = [] for raw in messages: if not isinstance(raw, dict): continue role = raw.get("role") if role not in _ROLES: continue msg: dict = {"role": role, "content": str(raw.get("content") or "")} if role == "tool": call_id = raw.get("toolCallId") or raw.get("tool_call_id") if not call_id: continue msg["tool_call_id"] = str(call_id) if raw.get("name"): msg["name"] = str(raw["name"]) calls = raw.get("toolCalls") or raw.get("tool_calls") if role == "assistant" and isinstance(calls, list) and calls: msg["tool_calls"] = [ { "id": str(c.get("id") or ""), "type": "function", "function": { "name": str(c.get("name") or ""), "arguments": _json_object(c.get("arguments")), }, } for c in calls if isinstance(c, dict) and c.get("name") ] out.append(msg) return out def _json_object(value) -> str: """A tool call's arguments, as something the router will accept. The router rejects the whole request with a 400 when any call's arguments are not valid JSON -- so a model that wrote a JavaScript object literal kills the conversation on the *next* turn, when its own call is echoed back, and the error names neither the call nor the turn it came from. The client repairs what it can before it gets here. This is the guarantee: what still cannot be parsed becomes an empty object, and the `tool` message answering it already says why. """ text = value if isinstance(value, str) else json.dumps(value or {}) text = text.strip() or "{}" try: parsed = json.loads(text) except (ValueError, TypeError): return "{}" # An array or a scalar is valid JSON and still not an argument list. return text if isinstance(parsed, dict) else "{}" def normalise_tools(tools) -> list: """Accept only well-formed function tools, and not many of them.""" if not isinstance(tools, list): return [] out = [] for t in tools[:MAX_TOOLS]: if not isinstance(t, dict): continue fn = t.get("function") if not isinstance(fn, dict) or not isinstance(fn.get("name"), str) or not fn["name"]: continue out.append({ "type": "function", "function": { "name": fn["name"], "description": str(fn.get("description") or ""), "parameters": fn.get("parameters") if isinstance(fn.get("parameters"), dict) else {}, }, }) return out def build_payload(body: dict, model: str, reasoning: bool) -> dict: """The request we send upstream, from the request the browser sent us.""" payload: dict = { "model": model, "messages": normalise_messages(body["messages"]), "stream": True, # Per-chunk usage is what feeds the live token readouts. Without it the # totals only arrive at the end, and the inspector sits empty. "stream_options": {"include_usage": True}, "max_tokens": int(body.get("maxTokens") or 4096), } if body.get("temperature") is not None: payload["temperature"] = float(body["temperature"]) tools = normalise_tools(body.get("tools")) if tools: payload["tools"] = tools payload["tool_choice"] = "auto" if reasoning: effort = body.get("reasoningEffort") # The model card: the parameter defaults to `max` and only `low` and # `high` need passing. Sending `max` explicitly is a no-op, so don't. if effort in ("low", "high"): payload["reasoning_effort"] = effort # Also from the card: chat scenarios should set this, or the model # carries prior turns' thinking back into context. payload["chat_template_kwargs"] = {"clear_thinking": True} return payload