Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 7,553 Bytes
89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | import { API_V1, CREDENTIALS, checkTenant, signal, UNAUTHORIZED_EVENT } from "../apiContract";
import { refusalPair, refusalSentence, type ChatResult } from "../query/queryApi";
type StreamLine = { message?: string; label?: string; name?: string; [key: string]: unknown };
type StreamError = StreamLine & { status?: number; code?: string; retry?: boolean; result?: unknown;
error?: { message?: string; code?: string } };
export type ChatStreamEvent =
| { type: "thinking"; data: StreamLine }
| { type: "tool_step"; data: StreamLine }
| { type: "text_delta"; data: string | { delta?: string; text?: string; content?: string } }
| { type: "joined"; data: Record<string, unknown> }
| { type: "paused"; data: Record<string, unknown> }
| { type: "done"; data: ChatResult }
| { type: "error"; data: StreamError };
export type ChatStreamResult = Extract<ChatStreamEvent, { type: "done" | "joined" }>;
export type ChatStreamRequest = { question: string; database: string; sources: string[]; model: string };
export class ChatStreamFailure extends Error {
readonly fallbackEligible: boolean;
readonly admitted: boolean;
/** C2: the server's refusal code, when the response or error event carried one. */
readonly code?: string;
constructor(message: string, fallbackEligible: boolean, admitted = false, code?: string) {
super(message);
this.fallbackEligible = fallbackEligible;
this.admitted = admitted;
if (code) this.code = code;
}
}
export function streamDelta(data: Extract<ChatStreamEvent, { type: "text_delta" }>["data"]): string {
return typeof data === "string" ? data : data.delta ?? data.text ?? data.content ?? "";
}
function boundaryFailure(status: number, code = ""): boolean {
return [401, 403, 409].includes(status) || /unauthorized|forbidden|refused|tenant|permission/i.test(code);
}
function completedResult(value: unknown): value is ChatResult {
if (!value || typeof value !== "object") return false;
const result = value as Partial<ChatResult>;
return typeof result.userMessage?.id === "string" && result.userMessage.id.trim().length > 0 &&
typeof result.message?.id === "string" && result.message.id.trim().length > 0 &&
result.userMessage.id !== result.message.id && result.userMessage.role === "user" &&
result.message.role === "assistant" && typeof result.message.threadId === "string" &&
result.message.threadId.length > 0 && result.userMessage.threadId === result.message.threadId &&
typeof result.message.content === "string" && Array.isArray(result.citations);
}
/** Read C3 records across arbitrary UTF-8 and SSE chunk boundaries. */
export async function streamChat(body: ChatStreamRequest, onEvent: (event: ChatStreamEvent) => void,
abortSignal?: AbortSignal): Promise<ChatStreamResult> {
abortSignal?.throwIfAborted();
let response: Response;
try {
response = await fetch(`${API_V1}/query/chat/stream`, {
method: "POST", credentials: CREDENTIALS, signal: abortSignal,
headers: { "Content-Type": "application/json", Accept: "text/event-stream" }, body: JSON.stringify(body),
});
} catch {
abortSignal?.throwIfAborted();
// Losing the response cannot establish that the server refused admission. Read its log.
throw new ChatStreamFailure(refusalSentence(), false, true);
}
abortSignal?.throwIfAborted();
if (checkTenant(response)) throw new ChatStreamFailure("The workspace changed. Open this chat again.", false);
if (response.status === 401) signal(UNAUTHORIZED_EVENT);
if (!response.ok) {
const detail = await response.json().catch(() => null) as StreamError | null;
const { code } = refusalPair(detail);
throw new ChatStreamFailure(refusalSentence(code),
!boundaryFailure(response.status, code), false, code);
}
if (response.headers.get("Content-Type")?.split(";")[0].trim().toLowerCase() !== "text/event-stream" || !response.body)
throw new ChatStreamFailure("The answer stream is unavailable.", true);
const reader = response.body.getReader();
const decoder = new TextDecoder();
// An accepted SSE response belongs to an admitted turn, even before its first frame.
let buffer = "", content = "", admitted = true;
let rejectAbort: (reason: unknown) => void = () => {};
const aborted = new Promise<never>((_, reject) => { rejectAbort = reject; });
const cancel = () => {
rejectAbort(abortSignal?.reason || new DOMException("Aborted", "AbortError"));
void reader.cancel().catch(() => {});
};
abortSignal?.addEventListener("abort", cancel, { once: true });
try {
abortSignal?.throwIfAborted();
for (;;) {
const chunk = await Promise.race([reader.read(), aborted]);
abortSignal?.throwIfAborted();
buffer += decoder.decode(chunk.value, { stream: !chunk.done });
let record: RegExpExecArray | null;
while ((record = /\r?\n\r?\n/.exec(buffer))) {
const lines = buffer.slice(0, record.index).split(/\r?\n/);
buffer = buffer.slice(record.index + record[0].length);
const type = lines.find((line) => line.startsWith("event:"))?.slice(6).trim();
const data = lines.filter((line) => line.startsWith("data:"))
.map((line) => line.slice(5).replace(/^ /, "")).join("\n");
if (!data || !type) continue;
if (!["thinking", "tool_step", "text_delta", "joined", "paused", "done", "error"].includes(type)) continue;
const event = { type, data: JSON.parse(data) } as ChatStreamEvent;
admitted = true;
if (event.type === "text_delta") content += streamDelta(event.data);
if (event.type === "done" && (!completedResult(event.data)
|| event.data.message.content !== content)) throw new ChatStreamFailure("The answer stream was incomplete.", false, admitted);
if (event.type === "error" && event.data.result) {
if (completedResult(event.data.result)) {
const recovered: ChatStreamResult = { type: "done", data: event.data.result };
onEvent(recovered);
return recovered;
}
const controlled = event.data.result as { status?: unknown };
if (controlled.status === "joined" || controlled.status === "stopped") {
const joined: ChatStreamResult = { type: "joined", data: { status: controlled.status } };
onEvent(joined);
return joined;
}
}
onEvent(event);
if (event.type === "error") {
const { code } = refusalPair(event.data);
throw new ChatStreamFailure(refusalSentence(code), !admitted && event.data.retry !== false &&
!boundaryFailure(event.data.status || 0, code), admitted, code);
}
if (event.type === "done" || event.type === "joined") return event;
}
if (chunk.done) throw new ChatStreamFailure("The answer stream ended before the answer was saved.", false, true);
}
} catch (error) {
abortSignal?.throwIfAborted();
if (error instanceof ChatStreamFailure) throw error;
throw new ChatStreamFailure(refusalSentence(), !admitted, admitted);
} finally {
abortSignal?.removeEventListener("abort", cancel);
void reader.cancel().catch(() => {});
reader.releaseLock();
}
}
export function canFallback(error: unknown, abortSignal?: AbortSignal): boolean {
if (abortSignal?.aborted || (error instanceof Error && error.name === "AbortError")) return false;
return error instanceof ChatStreamFailure ? error.fallbackEligible : error instanceof TypeError || error instanceof SyntaxError;
}
|