| import { Type } from "typebox"; |
| import { |
| COMPUTER_USE_V1_ACTION_NAMES, |
| type ComputerUseV2ActionName, |
| } from "../../plugins/computer-use-contract.js"; |
| import { |
| optionalFiniteNumberSchema, |
| optionalNonNegativeIntegerSchema, |
| optionalPositiveIntegerSchema, |
| optionalStringEnum, |
| stringEnum, |
| } from "../schema/typebox.js"; |
| import { MAX_HOLD_SECONDS, MAX_WAIT_SECONDS } from "./computer-tool-shared.js"; |
| import { gatewayCallOptionSchemaProperties } from "./gateway-schema.js"; |
|
|
| export const COMPUTER_TOOL_ACTIONS = COMPUTER_USE_V1_ACTION_NAMES; |
|
|
| const EXECUTION_OWNED_ACTIONS = new Set<ComputerUseV2ActionName>([ |
| "browser_set_input_files", |
| "browser_download", |
| "get_recording_state", |
| "start_recording", |
| "stop_recording", |
| "replay_trajectory", |
| ]); |
|
|
| export function availableComputerActions( |
| actions: readonly ComputerUseV2ActionName[], |
| hasCleanupOwner: boolean, |
| ): readonly ComputerUseV2ActionName[] { |
| const available = hasCleanupOwner |
| ? actions |
| : actions.filter((action) => !EXECUTION_OWNED_ACTIONS.has(action)); |
| |
| return available.includes("screenshot") && !available.includes("wait") |
| ? [...available, "wait"] |
| : available; |
| } |
|
|
| export function createComputerToolSchema( |
| actions: readonly ComputerUseV2ActionName[], |
| targetScope: "paired" | "session" = "paired", |
| ) { |
| const supportsHold = actions.includes("hold_key"); |
| const accessibilityTarget = actions.includes("get_window_state") |
| ? "get_window_state with windowRef" |
| : actions.includes("get_accessibility_tree") |
| ? "get_accessibility_tree" |
| : "Accessibility observations"; |
| return Type.Object({ |
| action: stringEnum(actions), |
| ...(targetScope === "paired" |
| ? { |
| ...gatewayCallOptionSchemaProperties(), |
| target: optionalStringEnum(["gateway", "node"] as const, { |
| description: |
| "Computer host. Defaults to the Gateway desktop when configured, otherwise a paired node. Later calls retain the selected host.", |
| }), |
| node: Type.Optional( |
| Type.String({ |
| description: |
| "Paired node id or display name; implies target=node. Omit when selecting the sole connected computer-capable node.", |
| }), |
| ), |
| } |
| : {}), |
| |
| |
| coordinate: Type.Optional( |
| Type.Array(Type.Integer({ minimum: 0 }), { |
| minItems: 2, |
| maxItems: 2, |
| description: "[x, y] target in pixels of the most recent screenshot.", |
| }), |
| ), |
| startCoordinate: Type.Optional( |
| Type.Array(Type.Integer({ minimum: 0 }), { |
| minItems: 2, |
| maxItems: 2, |
| description: "left_click_drag: [x, y] drag origin in screenshot pixels.", |
| }), |
| ), |
| destinationCoordinate: Type.Optional( |
| Type.Array(Type.Number({ minimum: 0 }), { |
| minItems: 2, |
| maxItems: 2, |
| description: "browser_pointer drag destination [x, y] in viewport CSS pixels.", |
| }), |
| ), |
| text: Type.Optional( |
| Type.String({ |
| description: |
| `type: text to type; ${supportsHold ? "key/hold_key" : "key"}: key combo such as "cmd+shift+t" or "Return"; ` + |
| 'click/scroll actions: modifier keys to hold ("shift", "ctrl", "alt", "cmd").', |
| }), |
| ), |
| scrollDirection: optionalStringEnum(["up", "down", "left", "right"] as const), |
| scrollAmount: optionalPositiveIntegerSchema({ |
| maximum: 100, |
| description: "scroll: number of wheel ticks.", |
| }), |
| duration: optionalFiniteNumberSchema({ |
| minimum: 0, |
| maximum: MAX_WAIT_SECONDS, |
| description: supportsHold |
| ? `Seconds. hold_key: >0 to ${MAX_HOLD_SECONDS}; wait: 0 to ${MAX_WAIT_SECONDS}.` |
| : `Seconds. wait: 0 to ${MAX_WAIT_SECONDS}; this does not extend a key tap.`, |
| }), |
| screenIndex: optionalNonNegativeIntegerSchema(), |
| frameId: Type.Optional( |
| Type.String({ |
| description: |
| "Desktop coordinate actions: exact frame id returned by the most recent screenshot result.", |
| }), |
| ), |
| windowRef: Type.Optional( |
| Type.String({ |
| description: |
| "Opaque window reference for window actions; not valid for screenshot or wait.", |
| }), |
| ), |
| browserRef: Type.Optional( |
| Type.String({ description: "Opaque browser reference from get_browser_state." }), |
| ), |
| pageRef: Type.Optional( |
| Type.String({ description: "Opaque browser page reference from get_browser_state." }), |
| ), |
| elementRef: Type.Optional( |
| Type.String({ description: "Opaque accessibility element reference from observation." }), |
| ), |
| observationId: Type.Optional( |
| Type.String({ |
| description: |
| "Window/browser input: observation id from the latest targeted observation; a desktop frameId cannot replace it.", |
| }), |
| ), |
| deliveryMode: optionalStringEnum(["background", "foreground"] as const, { |
| description: |
| "Window-targeted input delivery. This does not turn desktop input into background window input.", |
| }), |
| query: Type.Optional( |
| Type.String({ |
| description: |
| `${accessibilityTarget}: text filter.` + |
| (actions.includes("get_browser_state") |
| ? " get_browser_state: requires snapshotFormat=semantic_v2 with browserRef and pageRef." |
| : ""), |
| }), |
| ), |
| depth: Type.Optional( |
| Type.Integer({ |
| minimum: 0, |
| maximum: 64, |
| description: `${accessibilityTarget}: maximum tree depth.`, |
| }), |
| ), |
| maxElements: Type.Optional( |
| Type.Integer({ |
| minimum: 1, |
| maximum: 2_000, |
| description: `${accessibilityTarget}: maximum returned elements.`, |
| }), |
| ), |
| app: Type.Optional(Type.String()), |
| value: Type.Optional(Type.String()), |
| path: Type.Optional( |
| Type.Array(Type.String({ minLength: 1, maxLength: 200 }), { minItems: 1, maxItems: 16 }), |
| ), |
| x1: Type.Optional(Type.Number({ minimum: 0 })), |
| y1: Type.Optional(Type.Number({ minimum: 0 })), |
| x2: Type.Optional(Type.Number({ minimum: 0 })), |
| y2: Type.Optional(Type.Number({ minimum: 0 })), |
| reason: optionalStringEnum([ |
| "ax_tree_pixel_mismatch", |
| "background_delivery_failed", |
| "foreground_ineffective", |
| "no_window_target", |
| "other", |
| ] as const), |
| snapshotFormat: optionalStringEnum(["dom_refs_v1", "semantic_v2"] as const), |
| continuation: Type.Optional(Type.String()), |
| includeScreenshot: Type.Optional(Type.Boolean()), |
| profile: optionalStringEnum(["isolated_new", "isolated_named"] as const), |
| profileName: Type.Optional(Type.String({ minLength: 1, maxLength: 64 })), |
| url: Type.Optional(Type.String()), |
| inputRoute: optionalStringEnum(["trusted", "dom_event"] as const), |
| mode: optionalStringEnum(["insert_text", "keystrokes"] as const), |
| replace: Type.Optional(Type.Boolean()), |
| dialogAction: optionalStringEnum(["inspect", "accept", "dismiss"] as const), |
| dialogRef: Type.Optional(Type.String()), |
| promptText: Type.Optional(Type.String()), |
| resourceHandle: Type.Optional( |
| Type.String({ description: "Opaque host-owned Computer Use resource handle." }), |
| ), |
| resourceHandles: Type.Optional( |
| Type.Array(Type.String({ minLength: 1 }), { minItems: 1, maxItems: 32 }), |
| ), |
| recordVideo: Type.Optional(Type.Boolean()), |
| delayMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 10_000 })), |
| stopOnError: Type.Optional(Type.Boolean()), |
| pointerAction: optionalStringEnum([ |
| "hover", |
| "right_click", |
| "double_click", |
| "scroll", |
| "drag", |
| ] as const), |
| destinationElementRef: Type.Optional(Type.String()), |
| deltaX: Type.Optional(Type.Number()), |
| deltaY: Type.Optional(Type.Number()), |
| }); |
| } |
|
|
