import { Type } from "typebox"; import { Value } from "typebox/value"; import { beforeEach, describe, expect, it } from "vitest"; import { getImageMetadata } from "../../media/image-ops.js"; import { createSolidPngBuffer } from "../../plugin-sdk/test-helpers/image-fixtures.js"; import type { ComputerUseV2ActionName } from "../../plugins/computer-use-contract.js"; import { callGatewayToolMock, COMPUTER_ACT_COMMAND, type ComputerActBody, createVisionComputerTool, EFFECTIVE_REF_WIDTH, listNodesMock, macComputerNode, readActionEnum, readLastComputerActParams, resetComputerToolMocks, screenshotPayload, sleepMock, v2Descriptor, } from "./computer-tool.test-helpers.js"; // Frozen from v2026.9.4 src/plugins/computer-use-contract.ts, including actionObject fields. // Keep independent of the current schema so mixed-version regressions remain visible. const releasedWindowStateSchema = Type.Object( { action: Type.Enum(["get_window_state"], { type: "string" }), executionId: Type.Optional( Type.String({ pattern: "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", }), ), windowRef: Type.String({ minLength: 1 }), query: Type.Optional(Type.String()), depth: Type.Optional(Type.Integer({ minimum: 0, maximum: 64 })), maxElements: Type.Optional(Type.Integer({ minimum: 1, maximum: 2_000 })), }, { additionalProperties: false }, ); describe("createComputerTool v2 execution", () => { beforeEach(resetComputerToolMocks); it.each(["screenshot", "wait"] as const)( "rejects a targeted %s before desktop capture", async (action) => { listNodesMock.mockResolvedValue([ macComputerNode({ computerUse: v2Descriptor(["screenshot", "get_window_state", "get_browser_state"]), }), ]); const tool = createVisionComputerTool(); for (const reference of [ "windowRef", "browserRef", "pageRef", "elementRef", "observationId", ]) { await expect( tool.execute(reference, { action, [reference]: "target-1", duration: 0 }), ).rejects.toThrow(/COMPUTER_INVALID_REQUEST:.*get_window_state/); } expect(callGatewayToolMock).not.toHaveBeenCalled(); expect(sleepMock).not.toHaveBeenCalled(); }, ); it("derives local wait from the selected node screenshot capability", async () => { const actions: ComputerUseV2ActionName[] = ["screenshot", "list_apps", "get_window_state"]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); const tool = createVisionComputerTool(); expect(tool.description).not.toContain("get_window_state"); await tool.execute("select", { action: "wait", duration: 0 }); expect(readActionEnum(tool)).toEqual([...actions, "wait"]); expect(sleepMock).toHaveBeenCalledWith(0, undefined); expect( callGatewayToolMock.mock.calls.map((call) => (call[2] as ComputerActBody).command), ).toEqual(["screen.snapshot"]); expect(tool.description).toContain("Observe first with `get_window_state`"); }); it("adopts refreshed node capabilities on explicit re-selection without changing Gateways", async () => { const initial = v2Descriptor(["screenshot", "list_windows"]); const upgraded = v2Descriptor(["screenshot", "launch_app"], { provider: { ...initial.provider, generation: "generation-2" }, }); const gatewayOptions = { gatewayUrl: "wss://gateway.example", gatewayToken: "fixture-token" }; listNodesMock .mockResolvedValueOnce([macComputerNode({ computerUse: initial })]) .mockResolvedValue([macComputerNode({ computerUse: upgraded })]); const tool = createVisionComputerTool(); await tool.execute("before-upgrade", { action: "screenshot", node: "mac-1", ...gatewayOptions, }); expect(readActionEnum(tool)).toContain("list_windows"); expect(readActionEnum(tool)).not.toContain("launch_app"); await tool.execute("after-upgrade", { action: "screenshot", node: "mac-1" }); expect(listNodesMock).toHaveBeenLastCalledWith( expect.objectContaining(gatewayOptions), undefined, ); expect(readActionEnum(tool)).toContain("launch_app"); expect(readActionEnum(tool)).not.toContain("list_windows"); await tool.execute("new-action", { action: "launch_app", app: "Fixture" }); expect(callGatewayToolMock).toHaveBeenCalledWith( "node.invoke", expect.objectContaining(gatewayOptions), expect.objectContaining({ nodeId: "mac-1", command: "computer.act", params: expect.objectContaining({ action: "launch_app", app: "Fixture" }), }), { signal: undefined }, ); await expect( tool.execute("retarget", { action: "screenshot", node: "mac-1", gatewayUrl: "wss://other-gateway.example", }), ).rejects.toThrow("bound to its Gateway connection"); }); it("refreshes a prepared schema from the Gateway override target", async () => { const remoteCapabilities = v2Descriptor(["screenshot", "launch_app", "get_accessibility_tree"]); listNodesMock.mockResolvedValue([macComputerNode({ computerUse: remoteCapabilities })]); const tool = createVisionComputerTool({ pairedNodeComputerUse: { actions: ["screenshot", "list_windows", "get_window_state"], guidanceCapabilities: v2Descriptor(["screenshot", "list_windows", "get_window_state"]), }, }); expect(readActionEnum(tool)).toContain("list_windows"); expect(readActionEnum(tool)).not.toContain("launch_app"); for (const field of ["query", "depth", "maxElements"]) { expect(tool.parameters).toHaveProperty( `properties.${field}.description`, expect.stringContaining("get_window_state with windowRef"), ); } await tool.execute("remote-observe", { action: "screenshot", gatewayUrl: "wss://gateway.example", gatewayToken: "remote-token", }); expect(listNodesMock).toHaveBeenCalledWith( expect.objectContaining({ gatewayUrl: "wss://gateway.example", gatewayToken: "remote-token", }), undefined, ); expect(readActionEnum(tool)).toEqual([ "screenshot", "launch_app", "get_accessibility_tree", "wait", ]); for (const field of ["query", "depth", "maxElements"]) { expect(tool.parameters).toHaveProperty( `properties.${field}.description`, expect.stringContaining("get_accessibility_tree"), ); } await tool.execute("legacy-tree", { action: "get_accessibility_tree", query: "Save", depth: 10, maxElements: 100, }); expect(readLastComputerActParams()).toEqual({ action: "get_accessibility_tree", query: "Save", depth: 10, maxElements: 100, }); }); it("advertises execution-owned actions only with an attempt cleanup owner", async () => { const actions: ComputerUseV2ActionName[] = [ "screenshot", "browser_download", "start_recording", ]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); const withoutCleanup = createVisionComputerTool(); await withoutCleanup.execute("bind-without-cleanup", { action: "screenshot" }); expect(readActionEnum(withoutCleanup)).toEqual(["screenshot", "wait"]); const withCleanup = createVisionComputerTool({ registerRunCleanup: () => {} }); await withCleanup.execute("bind-with-cleanup", { action: "screenshot" }); expect(readActionEnum(withCleanup)).toEqual([...actions, "wait"]); }); it.each([ { name: "missing screenshot capability", actions: ["list_apps"], error: "does not advertise action wait", captures: 0, }, { name: "denied screenshot transport", actions: ["screenshot"], error: "snapshot policy denied", captures: 1, }, ] as const)("keeps $name authoritative for local wait", async ({ actions, error, captures }) => { listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor([...actions]) })]); callGatewayToolMock.mockRejectedValue(new Error("snapshot policy denied")); const tool = createVisionComputerTool(); await expect(tool.execute("wait", { action: "wait", duration: 0 })).rejects.toThrow(error); expect(callGatewayToolMock).toHaveBeenCalledTimes(captures); expect( callGatewayToolMock.mock.calls.every( (call) => (call[2] as ComputerActBody).command === "screen.snapshot", ), ).toBe(true); }); it.each([ ["landscape pixels", 1568, 784, "image-pixels", 784, 392], ["portrait pixels", 784, 1568, "image-pixels", 392, 784], ["rounded axes", 1567, 785, "image-pixels", 783.5, 391.846922], ["unscaled pixels", 400, 300, "image-pixels", 200, 150], ["global logical points", 1568, 784, "global-logical-points", 600, 300], ["provider-defined units", 1568, 784, { unit: "vendor-defined" }, 600, 300], ] as const)( "binds delivered window image coordinates: %s", async (_name, width, height, coordinateSpace, expectedX, expectedY) => { const actions: ComputerUseV2ActionName[] = [ "get_window_state", "left_click", "left_click_drag", "zoom", ]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); const bounds = { x: 1920, y: 300, width: 400, height: 200 }; const observation = { kind: "window", base64: createSolidPngBuffer(width, height, { r: 70, g: 125, b: 180 }).toString("base64"), format: "png", width, height, observationId: "observation-1", elements: [{ elementRef: "element-1", role: "button", label: "Save", bounds }], }; callGatewayToolMock.mockImplementation(async (_method, _opts, body) => { const request = body as ComputerActBody; if (request.command !== COMPUTER_ACT_COMMAND) { return screenshotPayload(); } return request.params?.action === "get_window_state" ? { payload: { ok: true, observation, details: { coordinateSpace } } } : { payload: { ok: true } }; }); const tool = createVisionComputerTool(); const refs = { windowRef: "window-1", observationId: "observation-1" }; const result = await tool.execute("observe", { action: "get_window_state", windowRef: refs.windowRef, }); const image = result.content.find((block) => block.type === "image"); if (!image) { throw new Error("Missing delivered observation image"); } const dimensions = await getImageMetadata(Buffer.from(image.data, "base64")); if (!dimensions) { throw new Error("Missing delivered image dimensions"); } expect(result.details).toMatchObject({ result: { observation: { ...dimensions, elements: [{ bounds }] } }, }); expect(callGatewayToolMock).toHaveBeenCalledOnce(); expect(sleepMock).not.toHaveBeenCalledWith(500, expect.anything()); const coordinate = [Math.floor(dimensions.width / 2), Math.floor(dimensions.height / 2)]; const startCoordinate = coordinate.map((value) => value / 2); for (const action of ["left_click", "left_click_drag", "zoom"] as const) { await tool.execute(action, { action, ...refs, ...(action === "zoom" ? { x1: startCoordinate[0], y1: startCoordinate[1], x2: coordinate[0], y2: coordinate[1], } : { coordinate, ...(action === "left_click_drag" ? { startCoordinate } : {}) }), }); const sent = readLastComputerActParams(action); expect(sent[action === "zoom" ? "x2" : "x"]).toBeCloseTo(expectedX, 6); expect(sent[action === "zoom" ? "y2" : "y"]).toBeCloseTo(expectedY, 6); if (action !== "left_click") { expect(sent[action === "zoom" ? "x1" : "fromX"]).toBeCloseTo(expectedX / 2, 6); expect(sent[action === "zoom" ? "y1" : "fromY"]).toBeCloseTo(expectedY / 2, 6); } } }, ); it.each([undefined, true])( "preserves capture on a released node with includeScreenshot=%s", async (includeScreenshot) => { listNodesMock.mockResolvedValue([ macComputerNode({ computerUse: v2Descriptor(["get_window_state"]) }), ]); callGatewayToolMock.mockImplementation(async (_method, _opts, body) => { const request = body as ComputerActBody; if ( request.command !== COMPUTER_ACT_COMMAND || !Value.Check(releasedWindowStateSchema, request.params) ) { throw new Error("Released node rejected get_window_state params"); } return { payload: { ok: true, observation: { kind: "window", base64: createSolidPngBuffer(2, 1, { r: 70, g: 125, b: 180 }).toString("base64"), format: "png", width: 2, height: 1, observationId: "released-observation", }, details: { coordinateSpace: "image-pixels" }, }, }; }); const tool = createVisionComputerTool(); const result = await tool.execute("observe", { action: "get_window_state", windowRef: "window-1", includeScreenshot, }); expect(result.content.some((block) => block.type === "image")).toBe(true); }, ); it.each(["true", null, 0])( "rejects invalid window screenshot option %s", async (includeScreenshot) => { listNodesMock.mockResolvedValue([ macComputerNode({ computerUse: v2Descriptor(["get_window_state"]) }), ]); const tool = createVisionComputerTool(); await expect( tool.execute("observe", { action: "get_window_state", windowRef: "window-1", includeScreenshot, }), ).rejects.toThrow("includeScreenshot must be a boolean"); expect(callGatewayToolMock).not.toHaveBeenCalled(); }, ); it.each(["invalid", "omitted"] as const)( "rejects pixel input for an %s observation image but keeps element refs", async (image) => { listNodesMock.mockResolvedValue([ macComputerNode({ computerUse: v2Descriptor(["get_window_state", "left_click"]) }), ]); callGatewayToolMock.mockResolvedValue({ payload: { ok: true, observation: { kind: "window", ...(image === "invalid" ? { base64: "invalid!" } : {}), width: 1568, height: 784, observationId: "observation-1", }, details: { coordinateSpace: "image-pixels" }, }, }); const tool = createVisionComputerTool(); const refs = { windowRef: "window-1", observationId: "observation-1" }; const result = await tool.execute("observe", { action: "get_window_state", windowRef: refs.windowRef, ...(image === "omitted" ? { includeScreenshot: false } : {}), }); if (image === "omitted") { expect(readLastComputerActParams()).toMatchObject({ includeScreenshot: false }); } expect(result.content.some((block) => block.type === "image")).toBe(false); callGatewayToolMock.mockClear(); await expect( tool.execute("pixels", { action: "left_click", ...refs, coordinate: [600, 300] }), ).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); expect(callGatewayToolMock).not.toHaveBeenCalled(); await tool.execute("element", { action: "left_click", ...refs, elementRef: "element-1" }); expect(readLastComputerActParams("left_click")).toMatchObject({ action: "left_click", elementRef: "element-1", }); }, ); it("rejects stale semantic references before dispatch", async () => { const actions: ComputerUseV2ActionName[] = ["get_window_state", "set_value"]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); callGatewayToolMock.mockResolvedValue({ payload: { ok: true, observation: { kind: "window", observationId: "observation-current" }, }, }); const tool = createVisionComputerTool(); await tool.execute("observe", { action: "get_window_state", windowRef: "window-1" }); callGatewayToolMock.mockClear(); await expect( tool.execute("write", { action: "set_value", windowRef: "window-1", elementRef: "element-1", observationId: "observation-stale", value: "hello", deliveryMode: "background", }), ).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); expect(callGatewayToolMock).not.toHaveBeenCalled(); }); it.each(["inspect", "accept", "dismiss"])( "captures an after-image only for a dialog mutation: %s", async (dialogAction) => { listNodesMock.mockResolvedValue([ macComputerNode({ computerUse: v2Descriptor(["browser_dialog"]) }), ]); callGatewayToolMock.mockImplementation(async (_method, _opts, body) => (body as ComputerActBody).command === COMPUTER_ACT_COMMAND ? { payload: { ok: true, effect: "confirmed" } } : screenshotPayload(), ); await createVisionComputerTool().execute("dialog", { action: "browser_dialog", browserRef: "browser-1", pageRef: "page-1", dialogAction, ...(dialogAction === "inspect" ? {} : { dialogRef: "dialog-1" }), }); expect( callGatewayToolMock.mock.calls.map((call) => (call[2] as ComputerActBody).command), ).toEqual( dialogAction === "inspect" ? [COMPUTER_ACT_COMMAND] : [COMPUTER_ACT_COMMAND, "screen.snapshot"], ); }, ); it("maps browser observations and opaque refs through the public tool", async () => { const actions: ComputerUseV2ActionName[] = ["get_browser_state", "browser_pointer"]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); callGatewayToolMock.mockResolvedValueOnce({ payload: { ok: true, observation: { kind: "browser", observationId: "browser-observation-1" }, details: { browserRef: "browser-1", pageRef: "page-1", elements: [{ elementRef: "element-1" }, { elementRef: "element-2" }], }, }, }); const tool = createVisionComputerTool(); await tool.execute("observe-browser", { action: "get_browser_state", browserRef: "browser-1", pageRef: "page-1", snapshotFormat: "dom_refs_v1", includeScreenshot: true, }); expect(readLastComputerActParams()).toEqual({ action: "get_browser_state", browserRef: "browser-1", pageRef: "page-1", snapshotFormat: "dom_refs_v1", includeScreenshot: true, }); expect(tool.parameters).toHaveProperty( "properties.query.description", expect.stringContaining("get_browser_state: requires snapshotFormat=semantic_v2"), ); callGatewayToolMock.mockImplementation(async (_method, _opts, body) => (body as ComputerActBody).command === COMPUTER_ACT_COMMAND ? { payload: { ok: true, effect: "confirmed" } } : screenshotPayload(), ); await tool.execute("drag-browser", { action: "browser_pointer", browserRef: "browser-1", pageRef: "page-1", observationId: "browser-observation-1", pointerAction: "drag", inputRoute: "dom_event", elementRef: "element-1", destinationElementRef: "element-2", }); expect(readLastComputerActParams()).toEqual({ action: "browser_pointer", browserRef: "browser-1", pageRef: "page-1", observationId: "browser-observation-1", pointerAction: "drag", inputRoute: "dom_event", elementRef: "element-1", destinationElementRef: "element-2", }); }); it("routes an observation-bound element click without requiring coordinates", async () => { const actions: ComputerUseV2ActionName[] = ["get_window_state", "left_click"]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); callGatewayToolMock.mockImplementation(async (_method, _opts, body) => { const request = body as ComputerActBody; if (request.command !== COMPUTER_ACT_COMMAND) { return screenshotPayload(); } if (request.params?.action === "get_window_state") { return { payload: { ok: true, observation: { kind: "window", observationId: "observation-1", }, }, }; } return { payload: { ok: true, effect: "confirmed" } }; }); const tool = createVisionComputerTool(); await tool.execute("observe", { action: "get_window_state", windowRef: "window-1" }); await expect( tool.execute("click", { action: "left_click", windowRef: "window-1", elementRef: "element-1", observationId: "observation-1", deliveryMode: "background", }), ).resolves.toBeDefined(); expect(readLastComputerActParams("left_click")).toEqual({ action: "left_click", screenIndex: 0, refWidth: EFFECTIVE_REF_WIDTH, windowRef: "window-1", elementRef: "element-1", observationId: "observation-1", deliveryMode: "background", }); }); it("maps the recording family through opaque resource parameters", async () => { const actions: ComputerUseV2ActionName[] = [ "get_recording_state", "start_recording", "stop_recording", "replay_trajectory", ]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); const tool = createVisionComputerTool({ registerRunCleanup: () => {} }); const resourceHandle = "openclaw:computer-resource:v1:123e4567-e89b-42d3-a456-426614174000"; await tool.execute("record", { action: "start_recording", recordVideo: true }); expect(readLastComputerActParams()).toEqual({ action: "start_recording", recordVideo: true }); await tool.execute("replay", { action: "replay_trajectory", resourceHandle, delayMs: 25, stopOnError: false, }); expect(readLastComputerActParams()).toEqual({ action: "replay_trajectory", resourceHandle, delayMs: 25, stopOnError: false, }); }); it("closes the exact host execution through attempt-owned cleanup", async () => { const actions: ComputerUseV2ActionName[] = ["start_recording"]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); let cleanup: ((reason: string) => Promise) | undefined; const tool = createVisionComputerTool({ registerRunCleanup: (registered) => { cleanup = registered; }, }); await tool.execute("record", { action: "start_recording" }); const start = callGatewayToolMock.mock.calls .map((call) => call[2] as ComputerActBody) .findLast((body) => body.command === COMPUTER_ACT_COMMAND); if (!start?.params) { throw new Error("missing start_recording node invocation"); } const executionId = start.params.executionId; expect(executionId).toEqual(expect.any(String)); await cleanup?.("completion"); const close = callGatewayToolMock.mock.calls.at(-1)?.[2] as ComputerActBody; expect(close.params).toEqual({ action: "__close_execution", executionId, reason: "completion", }); }); });