import { createHash } from "node:crypto"; import { expectDefined } from "@openclaw/normalization-core"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { getImageMetadata } from "../../media/image-ops.js"; import { createSolidPngBuffer } from "../../plugin-sdk/test-helpers/image-fixtures.js"; import type { ComputerActResult } from "../../plugins/computer-use-contract.js"; import type { ComputerToolTransport } from "./computer-tool-shared.js"; import { createVisionComputerTool, resetComputerToolMocks, sleepMock, TINY_PNG_BASE64, v2Descriptor, } from "./computer-tool.test-helpers.js"; const actionResult: ComputerActResult = { ok: true, effect: "confirmed", details: { route: "accessibility_action", deliveryMode: "background", evidence: "verified" }, }; function observationResult(index: number): ComputerActResult { return { ok: true, observation: { kind: "window", observationId: `observation-${index}`, elements: [ { elementRef: `element-${index}`, role: "button", label: "Save", bounds: { x: 0, y: 0, width: 1, height: 1 }, }, ], base64: TINY_PNG_BASE64, format: "png", width: 1, height: 1, }, details: { coordinateSpace: "image-pixels" }, }; } function createObservedTool( options: { inline?: boolean; followUp?: () => ComputerActResult; } = {}, ) { let observations = 0; let mutations = 0; const cache = new Map(); const computerUse = v2Descriptor(["get_window_state", "left_click"]); const invoke = vi.fn(async (request) => { if (request.command !== "computer.act") { throw new Error("Unexpected desktop capture"); } const key = request.idempotencyKey!; const cached = cache.get(key); if (cached) { return cached; } let result: ComputerActResult; if (request.commandParams.action === "get_window_state") { observations += 1; result = observations > 1 && options.followUp ? options.followUp() : observationResult(observations); } else { expect(request.commandParams).toMatchObject({ action: "left_click", windowRef: "window-1", observationId: `observation-${observations}`, }); if (request.commandParams.elementRef) { expect(request.commandParams.elementRef).toBe(`element-${observations}`); } mutations += 1; result = options.inline ? { ...observationResult(++observations), ...actionResult } : actionResult; } cache.set(key, result); return result; }); const tool = createVisionComputerTool({ idempotencyScope: "run-1", transport: { computerUse, resolveNode: async () => ({ nodeId: "desktop-1", computerUse }), invoke, }, }); const observe = () => tool.execute("observe", { action: "get_window_state", windowRef: "window-1" }); const click = (index: number, callId = `click-${index}`, signal?: AbortSignal) => tool.execute( callId, { action: "left_click", windowRef: "window-1", observationId: `observation-${index}`, elementRef: `element-${index}`, deliveryMode: "background", }, signal, ); return { tool, invoke, observe, click, counts: () => ({ observations, mutations }) }; } describe("computer targeted action observations", () => { beforeEach(resetComputerToolMocks); it("returns fresh refs and preserves input evidence without an extra model observation turn", async () => { const fixture = createObservedTool(); await fixture.observe(); const result = await fixture.click(1); expect(result.content.map((block) => block.type)).toEqual(["text", "text", "image"]); expect(result.content[0]).toEqual({ type: "text", text: JSON.stringify({ action: "left_click", ...actionResult }), }); expect(result.content[1]).toEqual({ type: "text", text: JSON.stringify({ action: "get_window_state", ...observationResult(2), observation: { ...observationResult(2).observation, base64: "[image]" }, }), }); expect(result.details).toMatchObject({ action: "left_click", result: actionResult, followUpObservation: { observation: { observationId: "observation-2" }, details: { coordinateSpace: "image-pixels" }, }, }); await fixture.click(2, "click-1:observation"); expect(fixture.counts()).toEqual({ observations: 3, mutations: 2 }); expect(sleepMock).toHaveBeenCalledTimes(2); expect(sleepMock).toHaveBeenCalledWith(500, undefined); const requests = fixture.invoke.mock.calls.map(([request]) => request); const mutationRequest = expectDefined(requests[1], "first input request"); const observationRequest = expectDefined(requests[2], "automatic observation request"); const mutationKey = `computer.act:v1:${createHash("sha256") .update(JSON.stringify(["run-1", "click-1", "computer.act"])) .digest("hex")}`; expect(mutationRequest.idempotencyKey).toBe(mutationKey); expect(observationRequest.idempotencyKey).toMatch(/^computer\.observation:v1:/); expect(new Set(requests.map((request) => request.idempotencyKey)).size).toBe(5); expect(observationRequest.commandParams).toEqual({ action: "get_window_state", executionId: mutationRequest.commandParams.executionId, windowRef: "window-1", }); await expect(fixture.click(1, "stale")).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); expect(fixture.counts().mutations).toBe(2); }); it("keeps direct and inline observations unchanged and avoids an additional read", async () => { const fixture = createObservedTool({ inline: true }); const direct = await fixture.observe(); expect(direct.content[0]).toEqual({ type: "text", text: JSON.stringify({ action: "get_window_state", ...observationResult(1), observation: { ...observationResult(1).observation, base64: "[image]" }, }), }); const inline = await fixture.click(1); expect(inline.content.map((block) => block.type)).toEqual(["text", "image"]); expect(inline.details).toMatchObject({ action: "left_click", result: { ...actionResult, observation: { observationId: "observation-2" } }, }); expect(inline.details).not.toHaveProperty("followUpObservation"); expect(fixture.invoke).toHaveBeenCalledTimes(2); expect(sleepMock).not.toHaveBeenCalled(); }); it.each(["image-pixels", "global-logical-points"] as const)( "binds the automatic observation's resized image using %s", async (coordinateSpace) => { const width = 1568; const height = 784; const fixture = createObservedTool({ followUp: () => ({ ...observationResult(2), observation: { ...observationResult(2).observation!, base64: createSolidPngBuffer(width, height, { r: 70, g: 125, b: 180 }).toString( "base64", ), width, height, }, details: { coordinateSpace }, }), }); await fixture.observe(); const result = await fixture.click(1); const image = result.content.find((block) => block.type === "image"); if (!image) { throw new Error("Missing automatic window observation image"); } const dimensions = await getImageMetadata(Buffer.from(image.data, "base64")); if (!dimensions) { throw new Error("Missing automatic window observation dimensions"); } expect(dimensions.width).toBeLessThan(width); const coordinate: [number, number] = [ Math.floor(dimensions.width / 2), Math.floor(dimensions.height / 2), ]; await fixture.tool.execute("next-pixel", { action: "left_click", windowRef: "window-1", observationId: "observation-2", coordinate, }); const [coordinateRequest] = expectDefined(fixture.invoke.mock.calls[3], "coordinate input"); const sent = coordinateRequest.commandParams; expect(sent.x).toBeCloseTo( coordinate[0] * (coordinateSpace === "image-pixels" ? width / dimensions.width : 1), ); expect(sent.y).toBeCloseTo( coordinate[1] * (coordinateSpace === "image-pixels" ? height / dimensions.height : 1), ); }, ); it.each([ [ "rejected", () => { throw new Error("window closed"); }, ], [ "read aborted", () => { throw new DOMException("read cancelled", "AbortError"); }, ], ["missing observation", () => ({ ok: true })], ["unsuccessful", () => ({ ok: false, details: { reason: "unavailable" } })], ] as const)( "preserves input success after a %s follow-up and expires old refs", async (_name, followUp) => { const fixture = createObservedTool({ followUp }); await fixture.observe(); const result = await fixture.click(1, "click", new AbortController().signal); expect(result.details).toMatchObject({ action: "left_click", result: actionResult }); expect(result.content[0]).toMatchObject({ text: expect.stringContaining("follow-up observation failed:"), }); await expect(fixture.click(1, "stale")).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); expect(fixture.counts()).toEqual({ observations: 2, mutations: 1 }); }, ); it("keeps caller cancellation authoritative without repeating input", async () => { const controller = new AbortController(); const fixture = createObservedTool({ followUp: () => { controller.abort(new Error("caller cancelled")); controller.signal.throwIfAborted(); return observationResult(2); }, }); await fixture.observe(); await expect(fixture.click(1, "click", controller.signal)).rejects.toThrow("caller cancelled"); expect(fixture.counts()).toEqual({ observations: 2, mutations: 1 }); }); it.each([ { name: "screen input", input: { action: "key", text: "Return" }, actions: ["key", "get_window_state"], }, { name: "unavailable window observation", input: { action: "key", text: "Return", windowRef: "window-1" }, actions: ["key"], }, { name: "browser preparation", input: { action: "browser_prepare", windowRef: "window-1" }, actions: ["browser_prepare", "get_window_state"], }, ] as const)("retains desktop capture for $name", async ({ input, actions }) => { const computerUse = v2Descriptor(["screenshot", ...actions]); const invoke = vi.fn(async (request) => request.command === "computer.act" ? actionResult : { format: "png", base64: TINY_PNG_BASE64, displayFrameId: "display-1", width: 1, height: 1, }, ); const tool = createVisionComputerTool({ transport: { computerUse, resolveNode: async () => ({ nodeId: "desktop-1", computerUse }), invoke, }, }); const result = await tool.execute("input", input); expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([ "computer.act", "screen.snapshot", ]); expect(result.content.some((block) => block.type === "image")).toBe(true); expect(sleepMock).toHaveBeenCalledWith(500, undefined); }); });