import type { ComputerUseCapabilityDescriptor, ComputerUseV2ActionName, } from "../../plugins/computer-use-contract.js"; export const COMPUTER_ACT_COMMAND = "computer.act"; export const SCREEN_SNAPSHOT_COMMAND = "screen.snapshot"; export const COMPUTER_REF_WIDTH = 1280; export const SCREENSHOT_QUALITY = 0.85; export const AFTER_ACTION_SCREENSHOT_DELAY_MS = 500; export const MAX_WAIT_SECONDS = 100; export const MAX_HOLD_SECONDS = 10; export const MODEL_OBSERVATION_MAX_ELEMENTS = 200; export type ComputerToolAction = ComputerUseV2ActionName; const COMPUTER_OBSERVATION_ACTIONS = new Set([ "screenshot", "wait", "list_apps", "list_windows", "get_accessibility_tree", "get_cursor_position", "get_window_state", "zoom", "get_browser_state", "get_recording_state", ] satisfies ComputerToolAction[]); // Observations may refresh frame/ref caches, but do not perform setup or input. // Keep result handling and replay classification on the same action contract. export function isComputerObservationAction( action: string | undefined, dialogAction?: unknown, ): boolean { return ( action !== undefined && (COMPUTER_OBSERVATION_ACTIONS.has(action) || (action === "browser_dialog" && dialogAction === "inspect")) ); } /** The owner binds the execution target and revalidates its authority before every dispatch. */ export type ComputerToolTransport = { readonly computerUse?: ComputerUseCapabilityDescriptor; resolveNode: ( query?: string, signal?: AbortSignal, ) => Promise<{ nodeId: string; computerUse?: ComputerUseCapabilityDescriptor }>; invoke: (params: { nodeId: string; command: typeof COMPUTER_ACT_COMMAND | typeof SCREEN_SNAPSHOT_COMMAND; commandParams: Record; timeoutMs?: number; idempotencyKey?: string; signal?: AbortSignal; }) => Promise; }; export type ComputerHost = { host: "gateway" } | { host: "node"; nodeId: string }; export type ComputerTarget = ComputerHost & { screenIndex: number }; export function computerHostKey(target: ComputerHost): string { return target.host === "gateway" ? "gateway" : `node:${target.nodeId}`; } export function computerTargetDetails(target: ComputerHost) { return target.host === "gateway" ? { target: "gateway" as const } : { node: target.nodeId }; } export type ComputerFrame = { target: ComputerTarget; id: string; displayFrameId: string; contextEpoch: number; }; export type ScreenshotCapture = { base64: string; displayFrameId: string; mimeType: string; width?: number; height?: number; }; export type ComputerObservationState = { targetKey: string; providerGeneration: string; observationId: string; imageCoordinates?: | { kind: "unavailable" } | { kind: "available"; scaleX: number; scaleY: number }; }; export type ComputerContextEpoch = { value: number; /** Tool result whose screenshot currently authorizes coordinates. */ frameToolCallId?: string; /** Digest of the exact sanitized image the model received for that result. */ frameImageIdentity?: string; }; export type ResolvedComputerTarget = { target: ComputerTarget; frame?: ComputerFrame; capabilities?: ComputerUseCapabilityDescriptor; };