| import type { AssistantMessage } from "@mariozechner/pi-ai"; |
| import { describe, expect, it } from "vitest"; |
| import { formatBillingErrorMessage } from "../../pi-embedded-helpers.js"; |
| import { makeAssistantMessageFixture } from "../../test-helpers/assistant-message-fixtures.js"; |
| import { |
| buildPayloads, |
| expectSinglePayloadText, |
| expectSingleToolErrorPayload, |
| } from "./payloads.test-helpers.js"; |
|
|
| describe("buildEmbeddedRunPayloads", () => { |
| const OVERLOADED_FALLBACK_TEXT = |
| "The AI service is temporarily overloaded. Please try again in a moment."; |
| const errorJson = |
| '{"type":"error","error":{"details":null,"type":"overloaded_error","message":"Overloaded"},"request_id":"req_011CX7DwS7tSvggaNHmefwWg"}'; |
| const errorJsonPretty = `{ |
| "type": "error", |
| "error": { |
| "details": null, |
| "type": "overloaded_error", |
| "message": "Overloaded" |
| }, |
| "request_id": "req_011CX7DwS7tSvggaNHmefwWg" |
| }`; |
| const makeAssistant = (overrides: Partial<AssistantMessage>): AssistantMessage => |
| makeAssistantMessageFixture({ |
| errorMessage: errorJson, |
| content: [{ type: "text", text: errorJson }], |
| ...overrides, |
| }); |
| const makeStoppedAssistant = () => |
| makeAssistant({ |
| stopReason: "stop", |
| errorMessage: undefined, |
| content: [], |
| }); |
|
|
| const expectOverloadedFallback = (payloads: ReturnType<typeof buildPayloads>) => { |
| expect(payloads).toHaveLength(1); |
| expect(payloads[0]?.text).toBe(OVERLOADED_FALLBACK_TEXT); |
| }; |
|
|
| function expectNoSyntheticCompletionForSession(sessionKey: string) { |
| const payloads = buildPayloads({ |
| sessionKey, |
| toolMetas: [{ toolName: "write", meta: "/tmp/out.md" }], |
| lastAssistant: makeAssistant({ |
| stopReason: "stop", |
| errorMessage: undefined, |
| content: [], |
| }), |
| }); |
| expect(payloads).toHaveLength(0); |
| } |
|
|
| it("suppresses raw API error JSON when the assistant errored", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: [errorJson], |
| lastAssistant: makeAssistant({}), |
| }); |
|
|
| expectOverloadedFallback(payloads); |
| expect(payloads[0]?.isError).toBe(true); |
| expect(payloads.some((payload) => payload.text === errorJson)).toBe(false); |
| }); |
|
|
| it("suppresses pretty-printed error JSON that differs from the errorMessage", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: [errorJsonPretty], |
| lastAssistant: makeAssistant({ errorMessage: errorJson }), |
| inlineToolResultsAllowed: true, |
| verboseLevel: "on", |
| }); |
|
|
| expectOverloadedFallback(payloads); |
| expect(payloads.some((payload) => payload.text === errorJsonPretty)).toBe(false); |
| }); |
|
|
| it("suppresses raw error JSON from fallback assistant text", () => { |
| const payloads = buildPayloads({ |
| lastAssistant: makeAssistant({ content: [{ type: "text", text: errorJsonPretty }] }), |
| }); |
|
|
| expectOverloadedFallback(payloads); |
| expect(payloads.some((payload) => payload.text?.includes("request_id"))).toBe(false); |
| }); |
|
|
| it("includes provider and model context for billing errors", () => { |
| const payloads = buildPayloads({ |
| lastAssistant: makeAssistant({ |
| model: "claude-3-5-sonnet", |
| errorMessage: "insufficient credits", |
| content: [{ type: "text", text: "insufficient credits" }], |
| }), |
| provider: "Anthropic", |
| model: "claude-3-5-sonnet", |
| }); |
|
|
| expect(payloads).toHaveLength(1); |
| expect(payloads[0]?.text).toBe(formatBillingErrorMessage("Anthropic", "claude-3-5-sonnet")); |
| expect(payloads[0]?.isError).toBe(true); |
| }); |
|
|
| it("does not emit a synthetic billing error for successful turns with stale errorMessage", () => { |
| const payloads = buildPayloads({ |
| lastAssistant: makeAssistant({ |
| stopReason: "stop", |
| errorMessage: "insufficient credits for embedding model", |
| content: [{ type: "text", text: "Handle payment required errors in your API." }], |
| }), |
| }); |
|
|
| expectSinglePayloadText(payloads, "Handle payment required errors in your API."); |
| }); |
|
|
| it("suppresses raw error JSON even when errorMessage is missing", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: [errorJsonPretty], |
| lastAssistant: makeAssistant({ errorMessage: undefined }), |
| }); |
|
|
| expect(payloads).toHaveLength(1); |
| expect(payloads[0]?.isError).toBe(true); |
| expect(payloads.some((payload) => payload.text?.includes("request_id"))).toBe(false); |
| }); |
|
|
| it("does not suppress error-shaped JSON when the assistant did not error", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: [errorJsonPretty], |
| lastAssistant: makeStoppedAssistant(), |
| }); |
|
|
| expectSinglePayloadText(payloads, errorJsonPretty.trim()); |
| }); |
|
|
| it("adds a fallback error when a tool fails and no assistant output exists", () => { |
| const payloads = buildPayloads({ |
| lastToolError: { toolName: "browser", error: "tab not found" }, |
| }); |
|
|
| expectSingleToolErrorPayload(payloads, { |
| title: "Browser", |
| absentDetail: "tab not found", |
| }); |
| }); |
|
|
| it("does not add tool error fallback when assistant output exists", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: ["All good"], |
| lastAssistant: makeStoppedAssistant(), |
| lastToolError: { toolName: "browser", error: "tab not found" }, |
| }); |
|
|
| expectSinglePayloadText(payloads, "All good"); |
| }); |
|
|
| it("does not add synthetic completion text when tools run without final assistant text", () => { |
| const payloads = buildPayloads({ |
| sessionKey: "agent:main:discord:direct:u123", |
| toolMetas: [{ toolName: "write", meta: "/tmp/out.md" }], |
| lastAssistant: makeStoppedAssistant(), |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("does not add synthetic completion text for channel sessions", () => { |
| expectNoSyntheticCompletionForSession("agent:main:discord:channel:c123"); |
| }); |
|
|
| it("does not add synthetic completion text for group sessions", () => { |
| expectNoSyntheticCompletionForSession("agent:main:telegram:group:g123"); |
| }); |
|
|
| it("does not add synthetic completion text when messaging tool already delivered output", () => { |
| const payloads = buildPayloads({ |
| sessionKey: "agent:main:discord:direct:u123", |
| toolMetas: [{ toolName: "message_send", meta: "sent to #ops" }], |
| didSendViaMessagingTool: true, |
| lastAssistant: makeAssistant({ |
| stopReason: "stop", |
| errorMessage: undefined, |
| content: [], |
| }), |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("suppresses non-mutating tool warnings when messaging tool already delivered the reply", () => { |
| const payloads = buildPayloads({ |
| toolMetas: [{ toolName: "message_send", meta: "sent to #ops" }], |
| didSendViaMessagingTool: true, |
| lastToolError: { toolName: "browser", error: "connection timeout" }, |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("does not add synthetic completion text when the run still has a tool error", () => { |
| const payloads = buildPayloads({ |
| toolMetas: [{ toolName: "browser", meta: "open https://example.com" }], |
| lastToolError: { toolName: "browser", error: "url required" }, |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("does not add synthetic completion text when no tools ran", () => { |
| const payloads = buildPayloads({ |
| lastAssistant: makeStoppedAssistant(), |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("adds tool error fallback when the assistant only invoked tools and verbose mode is on", () => { |
| const payloads = buildPayloads({ |
| lastAssistant: makeAssistant({ |
| stopReason: "toolUse", |
| errorMessage: undefined, |
| content: [ |
| { |
| type: "toolCall", |
| id: "toolu_01", |
| name: "exec", |
| arguments: { command: "echo hi" }, |
| }, |
| ], |
| }), |
| lastToolError: { toolName: "exec", error: "Command exited with code 1" }, |
| verboseLevel: "on", |
| }); |
|
|
| expectSingleToolErrorPayload(payloads, { |
| title: "Exec", |
| detail: "code 1", |
| }); |
| }); |
|
|
| it("does not add tool error fallback when assistant text exists after tool calls", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: ["Checked the page and recovered with final answer."], |
| lastAssistant: makeAssistant({ |
| stopReason: "toolUse", |
| errorMessage: undefined, |
| content: [ |
| { |
| type: "toolCall", |
| id: "toolu_01", |
| name: "browser", |
| arguments: { action: "search", query: "openclaw docs" }, |
| }, |
| ], |
| }), |
| lastToolError: { toolName: "browser", error: "connection timeout" }, |
| }); |
|
|
| expect(payloads).toHaveLength(1); |
| expect(payloads[0]?.isError).toBeUndefined(); |
| expect(payloads[0]?.text).toContain("recovered"); |
| }); |
|
|
| it("suppresses recoverable tool errors containing 'required' for non-mutating tools", () => { |
| const payloads = buildPayloads({ |
| lastToolError: { toolName: "browser", error: "url required" }, |
| }); |
|
|
| |
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("suppresses recoverable tool errors containing 'missing' for non-mutating tools", () => { |
| const payloads = buildPayloads({ |
| lastToolError: { toolName: "browser", error: "url missing" }, |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("suppresses recoverable tool errors containing 'invalid' for non-mutating tools", () => { |
| const payloads = buildPayloads({ |
| lastToolError: { toolName: "browser", error: "invalid parameter: url" }, |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("suppresses non-mutating non-recoverable tool errors when messages.suppressToolErrors is enabled", () => { |
| const payloads = buildPayloads({ |
| lastToolError: { toolName: "browser", error: "connection timeout" }, |
| config: { messages: { suppressToolErrors: true } }, |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it("suppresses mutating tool errors when suppressToolErrorWarnings is enabled", () => { |
| const payloads = buildPayloads({ |
| lastToolError: { toolName: "exec", error: "command not found" }, |
| suppressToolErrorWarnings: true, |
| }); |
|
|
| expect(payloads).toHaveLength(0); |
| }); |
|
|
| it.each([ |
| { |
| name: "still shows mutating tool errors when messages.suppressToolErrors is enabled", |
| payload: { |
| lastToolError: { toolName: "write", error: "connection timeout" }, |
| config: { messages: { suppressToolErrors: true } }, |
| }, |
| title: "Write", |
| absentDetail: "connection timeout", |
| }, |
| { |
| name: "shows recoverable tool errors for mutating tools", |
| payload: { |
| lastToolError: { toolName: "message", meta: "reply", error: "text required" }, |
| }, |
| title: "Message", |
| absentDetail: "required", |
| }, |
| { |
| name: "shows non-recoverable tool failure summaries to the user", |
| payload: { |
| lastToolError: { toolName: "browser", error: "connection timeout" }, |
| }, |
| title: "Browser", |
| absentDetail: "connection timeout", |
| }, |
| ])("$name", ({ payload, title, absentDetail }) => { |
| const payloads = buildPayloads(payload); |
| expectSingleToolErrorPayload(payloads, { title, absentDetail }); |
| }); |
|
|
| it("shows mutating tool errors even when assistant output exists", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: ["Done."], |
| lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, |
| lastToolError: { toolName: "write", error: "file missing" }, |
| }); |
|
|
| expect(payloads).toHaveLength(2); |
| expect(payloads[0]?.text).toBe("Done."); |
| expect(payloads[1]?.isError).toBe(true); |
| expect(payloads[1]?.text).toContain("Write"); |
| expect(payloads[1]?.text).not.toContain("missing"); |
| }); |
|
|
| it("does not treat session_status read failures as mutating when explicitly flagged", () => { |
| const payloads = buildPayloads({ |
| assistantTexts: ["Status loaded."], |
| lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, |
| lastToolError: { |
| toolName: "session_status", |
| error: "model required", |
| mutatingAction: false, |
| }, |
| }); |
|
|
| expect(payloads).toHaveLength(1); |
| expect(payloads[0]?.text).toBe("Status loaded."); |
| }); |
|
|
| it("dedupes identical tool warning text already present in assistant output", () => { |
| const seed = buildPayloads({ |
| lastToolError: { |
| toolName: "write", |
| error: "file missing", |
| mutatingAction: true, |
| }, |
| }); |
| const warningText = seed[0]?.text; |
| expect(warningText).toBeTruthy(); |
|
|
| const payloads = buildPayloads({ |
| assistantTexts: [warningText ?? ""], |
| lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, |
| lastToolError: { |
| toolName: "write", |
| error: "file missing", |
| mutatingAction: true, |
| }, |
| }); |
|
|
| expect(payloads).toHaveLength(1); |
| expect(payloads[0]?.text).toBe(warningText); |
| }); |
|
|
| it("includes non-recoverable tool error details when verbose mode is on", () => { |
| const payloads = buildPayloads({ |
| lastToolError: { toolName: "browser", error: "connection timeout" }, |
| verboseLevel: "on", |
| }); |
|
|
| expectSingleToolErrorPayload(payloads, { |
| title: "Browser", |
| detail: "connection timeout", |
| }); |
| }); |
| }); |
|
|
