import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce"; import type { ComputerActParams, ComputerUseCapabilityDescriptor, ComputerUseV2ActionName, } from "../../plugins/computer-use-contract.js"; import { COMPUTER_ACT_V1_ACTION_NAMES, COMPUTER_CONTRACT_MISMATCH, COMPUTER_STALE_OBSERVATION, COMPUTER_USE_V2_ACTION_NAMES, } from "../../plugins/computer-use-contract.js"; import { readFiniteNumberParam, readPositiveIntegerParam, readToolStringParam } from "./common.js"; import type { ComputerObservationState, ComputerToolAction } from "./computer-tool-shared.js"; import { COMPUTER_REF_WIDTH, MAX_HOLD_SECONDS } from "./computer-tool-shared.js"; const LOCAL_ACTIONS = new Set(["screenshot", "wait"]); const INPUT_ACTIONS = new Set( COMPUTER_USE_V2_ACTION_NAMES.filter((action) => !LOCAL_ACTIONS.has(action)), ); const ELEMENT_TARGETABLE_CLICK_ACTIONS = new Set([ "left_click", "right_click", "middle_click", "double_click", "triple_click", ]); const COORDINATE_REQUIRED_ACTIONS = new Set([ ...ELEMENT_TARGETABLE_CLICK_ACTIONS, "mouse_move", "left_click_drag", ]); const COORDINATE_OPTIONAL_ACTIONS = new Set([ "scroll", "left_mouse_down", "left_mouse_up", ]); const MODIFIER_TEXT_ACTIONS = new Set([ ...ELEMENT_TARGETABLE_CLICK_ACTIONS, "left_mouse_down", "left_mouse_up", "scroll", ]); const POINTER_OR_KEYBOARD_ACTIONS = new Set(COMPUTER_ACT_V1_ACTION_NAMES); const ESCALATION_REASONS = new Set([ "ax_tree_pixel_mismatch", "background_delivery_failed", "foreground_ineffective", "no_window_target", "other", ]); const SCROLL_DIRECTIONS = ["up", "down", "left", "right"] as const; function isScrollDirection(value: string): value is (typeof SCROLL_DIRECTIONS)[number] { return SCROLL_DIRECTIONS.some((direction) => direction === value); } export function isComputerActAction(action: ComputerToolAction): boolean { return INPUT_ACTIONS.has(action); } export function computerActionNeedsFrame( action: ComputerToolAction, input: Record, ): boolean { return ( !input.windowRef && !input.elementRef && (COORDINATE_REQUIRED_ACTIONS.has(action) || (COORDINATE_OPTIONAL_ACTIONS.has(action) && Array.isArray(input.coordinate))) ); } function readCoordinate( params: Record, key: "coordinate" | "startCoordinate", ): [number, number] | undefined { const raw = params[key]; if (raw === undefined) { return undefined; } if ( !Array.isArray(raw) || raw.length !== 2 || raw.some( (entry) => typeof entry !== "number" || !Number.isFinite(entry) || !Number.isInteger(entry) || entry < 0, ) ) { throw new Error(`${key} must be a pair of non-negative integers`); } return [raw[0] as number, raw[1] as number]; } function requireCoordinate(params: Record, action: string): [number, number] { const coordinate = readCoordinate(params, "coordinate"); if (!coordinate) { throw new Error(`coordinate [x, y] required for ${action}`); } return coordinate; } function readModifiers(params: Record, action: ComputerToolAction) { if (!MODIFIER_TEXT_ACTIONS.has(action)) { return undefined; } const text = typeof params.text === "string" ? params.text.trim() : ""; return text ? text : undefined; } function copyOptionalStringParam( target: Record, input: Record, key: string, ): void { const value = readToolStringParam(input, key); if (value !== undefined) { target[key] = value; } } function copyOptionalIntegerParam( target: Record, input: Record, key: string, bounds: { min: number; max: number }, ): void { const value = readFiniteNumberParam(input, key, bounds); if (value === undefined) { return; } if (!Number.isInteger(value)) { throw new Error(`${key} must be an integer`); } target[key] = value; } function copyDeliveryMode(target: Record, input: Record): void { const deliveryMode = normalizeOptionalLowercaseString(input.deliveryMode); if (deliveryMode === undefined) { return; } if (deliveryMode !== "background" && deliveryMode !== "foreground") { throw new Error("deliveryMode must be background or foreground"); } target.deliveryMode = deliveryMode; } function copyOptionalBooleanParam( target: Record, input: Record, key: string, ): void { const value = input[key]; if (value === undefined) { return; } if (typeof value !== "boolean") { throw new Error(`${key} must be a boolean`); } target[key] = value; } function copyBrowserRefs(target: Record, input: Record): void { target.browserRef = readToolStringParam(input, "browserRef", { required: true }); target.pageRef = readToolStringParam(input, "pageRef", { required: true }); } /** Builds the computer.act wire params for one tool input action. */ export function buildComputerActParams(params: { action: ComputerToolAction; input: Record; executionId: string; screenIndex: number; displayFrameId?: string; refWidth?: number; }): ComputerActParams { const { action, input } = params; const wire: Record = { action, executionId: params.executionId }; if (POINTER_OR_KEYBOARD_ACTIONS.has(action)) { wire.screenIndex = params.screenIndex; wire.refWidth = params.refWidth ?? COMPUTER_REF_WIDTH; } const elementRef = readToolStringParam(input, "elementRef"); if ( COORDINATE_REQUIRED_ACTIONS.has(action) && !(elementRef && ELEMENT_TARGETABLE_CLICK_ACTIONS.has(action)) ) { const [x, y] = requireCoordinate(input, action); wire.x = x; wire.y = y; } else if (COORDINATE_OPTIONAL_ACTIONS.has(action)) { const coordinate = readCoordinate(input, "coordinate"); if (coordinate) { wire.x = coordinate[0]; wire.y = coordinate[1]; } } if ((wire.x !== undefined || wire.fromX !== undefined) && params.displayFrameId) { wire.displayFrameId = params.displayFrameId; } const modifiers = readModifiers(input, action); if (modifiers) { wire.modifiers = modifiers; } switch (action) { case "left_click_drag": { const start = readCoordinate(input, "startCoordinate"); if (!start) { throw new Error("startCoordinate [x, y] required for left_click_drag"); } wire.fromX = start[0]; wire.fromY = start[1]; break; } case "scroll": { const direction = normalizeOptionalLowercaseString(input.scrollDirection); if (!direction || !isScrollDirection(direction)) { throw new Error("scrollDirection up|down|left|right required for scroll"); } wire.scrollDirection = direction; const amount = readPositiveIntegerParam(input, "scrollAmount") ?? 3; wire.scrollAmount = Math.min(100, amount); break; } case "type": { const text = typeof input.text === "string" ? input.text : ""; if (!text) { throw new Error("text required for type"); } wire.text = text; break; } case "key": case "hold_key": { const keys = readToolStringParam(input, "text", { required: true }); wire.keys = keys; if (action === "hold_key") { const seconds = readFiniteNumberParam(input, "duration", { min: 0, minExclusive: true, max: MAX_HOLD_SECONDS, message: `duration must be >0 and <=${MAX_HOLD_SECONDS} seconds for hold_key`, }) ?? 1; wire.durationMs = Math.round(seconds * 1000); } break; } case "get_accessibility_tree": case "get_window_state": { const windowRef = readToolStringParam(input, "windowRef", { required: action === "get_window_state", }); if (windowRef !== undefined) { wire.windowRef = windowRef; } if (action === "get_window_state") { copyOptionalBooleanParam(wire, input, "includeScreenshot"); // Released nodes capture by default but reject the newer wire field. if (wire.includeScreenshot === true) { delete wire.includeScreenshot; } } copyOptionalStringParam(wire, input, "query"); copyOptionalIntegerParam(wire, input, "depth", { min: 0, max: 64 }); copyOptionalIntegerParam(wire, input, "maxElements", { min: 1, max: 2_000 }); break; } case "launch_app": case "kill_app": { wire.app = readToolStringParam(input, "app", { required: true }); break; } case "bring_to_front": { wire.windowRef = readToolStringParam(input, "windowRef", { required: true }); break; } case "set_value": { for (const key of ["windowRef", "elementRef", "observationId", "value"] as const) { wire[key] = readToolStringParam(input, key, { required: true, allowEmpty: key === "value", }); } copyDeliveryMode(wire, input); break; } case "invoke_menu": { wire.windowRef = readToolStringParam(input, "windowRef", { required: true }); const path = input.path; if ( !Array.isArray(path) || path.length < 1 || path.length > 16 || path.some((segment) => typeof segment !== "string" || !segment.trim()) ) { throw new Error("path must contain 1-16 non-empty menu labels"); } wire.path = path; copyDeliveryMode(wire, input); break; } case "zoom": { wire.windowRef = readToolStringParam(input, "windowRef", { required: true }); wire.observationId = readToolStringParam(input, "observationId", { required: true }); for (const key of ["x1", "y1", "x2", "y2"] as const) { const value = readFiniteNumberParam(input, key, { min: 0 }); if (value === undefined) { throw new Error(`${key} required for zoom`); } wire[key] = value; } break; } case "get_browser_state": { const windowRef = readToolStringParam(input, "windowRef"); if (windowRef) { wire.windowRef = windowRef; break; } copyBrowserRefs(wire, input); for (const key of [ "snapshotFormat", "elementRef", "observationId", "query", "continuation", ] as const) { copyOptionalStringParam(wire, input, key); } copyOptionalBooleanParam(wire, input, "includeScreenshot"); break; } case "browser_prepare": { wire.windowRef = readToolStringParam(input, "windowRef", { required: true }); copyOptionalStringParam(wire, input, "profile"); copyOptionalStringParam(wire, input, "profileName"); break; } case "browser_navigate": { copyBrowserRefs(wire, input); wire.url = readToolStringParam(input, "url", { required: true }); break; } case "browser_click": { copyBrowserRefs(wire, input); wire.observationId = readToolStringParam(input, "observationId", { required: true }); copyOptionalStringParam(wire, input, "elementRef"); copyOptionalStringParam(wire, input, "inputRoute"); const coordinate = readCoordinate(input, "coordinate"); if (coordinate) { wire.x = coordinate[0]; wire.y = coordinate[1]; } break; } case "browser_type": { copyBrowserRefs(wire, input); for (const key of ["observationId", "elementRef"] as const) { wire[key] = readToolStringParam(input, key, { required: true }); } wire.text = readToolStringParam(input, "text", { required: true, allowEmpty: true }); copyOptionalStringParam(wire, input, "mode"); copyOptionalBooleanParam(wire, input, "replace"); break; } case "browser_dialog": { copyBrowserRefs(wire, input); wire.dialogAction = readToolStringParam(input, "dialogAction", { required: true }); copyOptionalStringParam(wire, input, "dialogRef"); copyOptionalStringParam(wire, input, "promptText"); copyDeliveryMode(wire, input); break; } case "browser_set_input_files": { copyBrowserRefs(wire, input); for (const key of ["observationId", "elementRef"] as const) { wire[key] = readToolStringParam(input, key, { required: true }); } const resourceHandles = input.resourceHandles; if ( !Array.isArray(resourceHandles) || resourceHandles.length < 1 || resourceHandles.length > 32 || resourceHandles.some((handle) => typeof handle !== "string" || !handle) ) { throw new Error("resourceHandles must contain 1-32 opaque resource handles"); } wire.resourceHandles = resourceHandles; break; } case "browser_download": { copyBrowserRefs(wire, input); for (const key of ["observationId", "elementRef"] as const) { wire[key] = readToolStringParam(input, key, { required: true }); } break; } case "browser_pointer": { copyBrowserRefs(wire, input); wire.observationId = readToolStringParam(input, "observationId", { required: true }); wire.pointerAction = readToolStringParam(input, "pointerAction", { required: true }); for (const key of ["inputRoute", "elementRef", "destinationElementRef"] as const) { copyOptionalStringParam(wire, input, key); } const coordinate = readCoordinate(input, "coordinate"); if (coordinate) { wire.x = coordinate[0]; wire.y = coordinate[1]; } const destination = input.destinationCoordinate; if (destination !== undefined) { if ( !Array.isArray(destination) || destination.length !== 2 || destination.some((value) => typeof value !== "number" || !Number.isFinite(value)) ) { throw new Error("destinationCoordinate must be a pair of finite numbers"); } wire.toX = destination[0]; wire.toY = destination[1]; } for (const key of ["deltaX", "deltaY"] as const) { const value = readFiniteNumberParam(input, key); if (value !== undefined) { wire[key] = value; } } break; } case "escalate_scope": { const reason = readToolStringParam(input, "reason", { required: true }); if (!ESCALATION_REASONS.has(reason)) { throw new Error("reason must be a supported escalation reason"); } wire.reason = reason; break; } case "start_recording": { copyOptionalBooleanParam(wire, input, "recordVideo"); break; } case "replay_trajectory": { wire.resourceHandle = readToolStringParam(input, "resourceHandle", { required: true }); copyOptionalIntegerParam(wire, input, "delayMs", { min: 0, max: 10_000 }); copyOptionalBooleanParam(wire, input, "stopOnError"); break; } default: break; } if (POINTER_OR_KEYBOARD_ACTIONS.has(action)) { for (const key of ["windowRef", "elementRef", "observationId"] as const) { copyOptionalStringParam(wire, input, key); } copyDeliveryMode(wire, input); } return wire as ComputerActParams; } export function validateCapabilityBoundInput(params: { action: ComputerUseV2ActionName; input: Record; targetKey: string; capabilities?: ComputerUseCapabilityDescriptor; observationState?: ComputerObservationState; }): void { const { capabilities, input } = params; const windowRef = readToolStringParam(input, "windowRef"); const browserRef = readToolStringParam(input, "browserRef"); const pageRef = readToolStringParam(input, "pageRef"); const elementRef = readToolStringParam(input, "elementRef"); const observationId = readToolStringParam(input, "observationId"); const deliveryMode = normalizeOptionalLowercaseString(input.deliveryMode); if ( LOCAL_ACTIONS.has(params.action) && (windowRef || browserRef || pageRef || elementRef || observationId) ) { const observations = ["get_window_state", "get_browser_state"].filter((action) => capabilities?.actions.some((available) => available === action), ); throw new Error( `COMPUTER_INVALID_REQUEST: ${params.action} captures a desktop screen; remove target and observation references.` + (observations.length ? ` Use ${observations.join(" or ")} for a targeted observation.` : ""), ); } if (windowRef && !capabilities?.targets.includes("window")) { throw new Error( `${COMPUTER_CONTRACT_MISMATCH}: selected computer has no window target support`, ); } if (elementRef && !capabilities?.targets.includes("element")) { throw new Error( `${COMPUTER_CONTRACT_MISMATCH}: selected computer has no element target support`, ); } if ((browserRef || pageRef) && !capabilities?.targets.includes("browser")) { throw new Error( `${COMPUTER_CONTRACT_MISMATCH}: selected computer has no browser target support`, ); } if (deliveryMode && !capabilities?.deliveryModes.some((mode) => mode === deliveryMode)) { throw new Error( `${COMPUTER_CONTRACT_MISMATCH}: selected computer does not advertise ${deliveryMode} delivery`, ); } if (elementRef && !observationId) { throw new Error(`${COMPUTER_STALE_OBSERVATION}: elementRef requires observationId`); } if (!observationId) { return; } if ( !params.observationState || params.observationState.targetKey !== params.targetKey || params.observationState.providerGeneration !== capabilities?.provider.generation || params.observationState.observationId !== observationId ) { throw new Error(`${COMPUTER_STALE_OBSERVATION}: take a fresh observation and retry`); } }