File size: 5,289 Bytes
eb3f11e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | import { vi } from "vitest";
import type {
ComputerUseCapabilityDescriptor,
ComputerUseV2ActionName,
} from "../../plugins/computer-use-contract.js";
const mocks = vi.hoisted(() => ({
listNodesMock: vi.fn(),
callGatewayToolMock: vi.fn(),
sleepMock: vi.fn(),
gatewayComputerStatusMock: vi.fn(),
}));
export const listNodesMock = mocks.listNodesMock;
export const callGatewayToolMock = mocks.callGatewayToolMock;
export const sleepMock = mocks.sleepMock;
export const gatewayComputerStatusMock = mocks.gatewayComputerStatusMock;
export const TINY_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=";
export const COMPUTER_ACT_COMMAND = "computer.act";
vi.mock("./nodes-utils.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("./nodes-utils.js")>();
return { ...actual, listNodes: listNodesMock };
});
vi.mock("./gateway.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("./gateway.js")>();
return { ...actual, callGatewayTool: callGatewayToolMock };
});
vi.mock("../../utils/sleep.js", () => ({ sleep: sleepMock }));
vi.mock("./computer-tool-gateway.js", () => ({
loadGatewayComputerStatus: gatewayComputerStatusMock,
bindGatewayComputerCleanup: async () => undefined,
}));
export const { createComputerTool, invalidateComputerFrameIfMissing } =
await import("./computer-tool.js");
export const { loadPairedComputerUseAvailabilityForSurface } =
await import("../computer-use-node-capabilities.js");
const { DEFAULT_IMAGE_MAX_DIMENSION_PX } = await import("../image-sanitization.js");
// With no config the reference width is capped at the default sanitization limit.
export const EFFECTIVE_REF_WIDTH = Math.min(1280, DEFAULT_IMAGE_MAX_DIMENSION_PX);
export function macComputerNode(overrides?: Record<string, unknown>) {
return {
nodeId: "mac-1",
displayName: "Studio",
platform: "macos",
connected: true,
commands: ["screen.snapshot", "computer.act"],
...overrides,
};
}
export function v2Descriptor(
actions: ComputerUseV2ActionName[],
overrides: Partial<ComputerUseCapabilityDescriptor> = {},
): ComputerUseCapabilityDescriptor {
return {
contractVersion: 2 as const,
provider: { id: "fixture", label: "Fixture", generation: "generation-1" },
actions,
targets: ["screen", "window", "element", "browser"] as const,
deliveryModes: ["background", "foreground"] as const,
observations: ["image", "accessibility", "browser"] as const,
features: { recording: false, agentCursor: false, multiDisplay: false },
...overrides,
};
}
export type ComputerTool = ReturnType<typeof createComputerTool>;
export type ComputerToolOptions = NonNullable<Parameters<typeof createComputerTool>[0]>;
export type ComputerActBody = {
nodeId?: string;
command?: string;
idempotencyKey?: string;
params?: Record<string, unknown>;
};
export function readActionEnum(tool: ComputerTool): string[] {
const schema = tool.parameters as { properties?: { action?: { enum?: string[] } } };
return schema.properties?.action?.enum ?? [];
}
export function screenshotPayload(screenIndex = 0, base64 = TINY_PNG_BASE64) {
return {
payload: {
format: "png",
base64,
displayFrameId: `display-${screenIndex}-frame`,
width: 1280,
height: 800,
screenIndex,
},
};
}
export function readFrameId(result: { details?: unknown }): string {
const frameId = (result.details as { frameId?: unknown } | undefined)?.frameId;
if (typeof frameId !== "string") {
throw new Error("missing frameId");
}
return frameId;
}
export function readLastComputerActParams(
action?: ComputerUseV2ActionName,
): Record<string, unknown> {
const call = callGatewayToolMock.mock.calls.findLast((entry) => {
const body = entry[2] as ComputerActBody;
return body.command === COMPUTER_ACT_COMMAND && (!action || body.params?.action === action);
});
const body = call?.[2] as { params?: Record<string, unknown> } | undefined;
if (!body?.params) {
throw new Error("missing computer.act request");
}
const { executionId: _executionId, ...params } = body.params;
return params;
}
export function createVisionComputerTool(options: ComputerToolOptions = {}) {
return createComputerTool({ modelHasVision: true, ...options });
}
export function resetComputerToolMocks() {
listNodesMock.mockReset();
callGatewayToolMock.mockReset();
gatewayComputerStatusMock.mockReset();
gatewayComputerStatusMock.mockResolvedValue({ configured: false, available: false });
sleepMock.mockReset();
sleepMock.mockImplementation((ms: number, signal?: AbortSignal) => {
if (signal?.aborted) {
return Promise.reject(new Error("Aborted"));
}
if (ms === 500 || !signal) {
return Promise.resolve();
}
return new Promise<void>((_resolve, reject) => {
signal.addEventListener("abort", () => reject(new Error("Aborted")), { once: true });
});
});
listNodesMock.mockResolvedValue([macComputerNode()]);
callGatewayToolMock.mockImplementation(async (_method, _opts, body) =>
(body as ComputerActBody).command === COMPUTER_ACT_COMMAND
? { payload: { ok: true } }
: screenshotPayload(),
);
}
|