File size: 7,952 Bytes
eb3f11e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 | import { Type } from "typebox";
import {
COMPUTER_USE_V1_ACTION_NAMES,
type ComputerUseV2ActionName,
} from "../../plugins/computer-use-contract.js";
import {
optionalFiniteNumberSchema,
optionalNonNegativeIntegerSchema,
optionalPositiveIntegerSchema,
optionalStringEnum,
stringEnum,
} from "../schema/typebox.js";
import { MAX_HOLD_SECONDS, MAX_WAIT_SECONDS } from "./computer-tool-shared.js";
import { gatewayCallOptionSchemaProperties } from "./gateway-schema.js";
export const COMPUTER_TOOL_ACTIONS = COMPUTER_USE_V1_ACTION_NAMES;
const EXECUTION_OWNED_ACTIONS = new Set<ComputerUseV2ActionName>([
"browser_set_input_files",
"browser_download",
"get_recording_state",
"start_recording",
"stop_recording",
"replay_trajectory",
]);
export function availableComputerActions(
actions: readonly ComputerUseV2ActionName[],
hasCleanupOwner: boolean,
): readonly ComputerUseV2ActionName[] {
const available = hasCleanupOwner
? actions
: actions.filter((action) => !EXECUTION_OWNED_ACTIONS.has(action));
// Local pacing uses snapshot authority; providers need no native wait action.
return available.includes("screenshot") && !available.includes("wait")
? [...available, "wait"]
: available;
}
export function createComputerToolSchema(
actions: readonly ComputerUseV2ActionName[],
targetScope: "paired" | "session" = "paired",
) {
const supportsHold = actions.includes("hold_key");
const accessibilityTarget = actions.includes("get_window_state")
? "get_window_state with windowRef"
: actions.includes("get_accessibility_tree")
? "get_accessibility_tree"
: "Accessibility observations";
return Type.Object({
action: stringEnum(actions),
...(targetScope === "paired"
? {
...gatewayCallOptionSchemaProperties(),
target: optionalStringEnum(["gateway", "node"] as const, {
description:
"Computer host. Defaults to the Gateway desktop when configured, otherwise a paired node. Later calls retain the selected host.",
}),
node: Type.Optional(
Type.String({
description:
"Paired node id or display name; implies target=node. Omit when selecting the sole connected computer-capable node.",
}),
),
}
: {}),
// Codex accepts a single schema in array `items`, not tuple item arrays.
// Fixed bounds preserve the coordinate-pair contract across runtimes.
coordinate: Type.Optional(
Type.Array(Type.Integer({ minimum: 0 }), {
minItems: 2,
maxItems: 2,
description: "[x, y] target in pixels of the most recent screenshot.",
}),
),
startCoordinate: Type.Optional(
Type.Array(Type.Integer({ minimum: 0 }), {
minItems: 2,
maxItems: 2,
description: "left_click_drag: [x, y] drag origin in screenshot pixels.",
}),
),
destinationCoordinate: Type.Optional(
Type.Array(Type.Number({ minimum: 0 }), {
minItems: 2,
maxItems: 2,
description: "browser_pointer drag destination [x, y] in viewport CSS pixels.",
}),
),
text: Type.Optional(
Type.String({
description:
`type: text to type; ${supportsHold ? "key/hold_key" : "key"}: key combo such as "cmd+shift+t" or "Return"; ` +
'click/scroll actions: modifier keys to hold ("shift", "ctrl", "alt", "cmd").',
}),
),
scrollDirection: optionalStringEnum(["up", "down", "left", "right"] as const),
scrollAmount: optionalPositiveIntegerSchema({
maximum: 100,
description: "scroll: number of wheel ticks.",
}),
duration: optionalFiniteNumberSchema({
minimum: 0,
maximum: MAX_WAIT_SECONDS,
description: supportsHold
? `Seconds. hold_key: >0 to ${MAX_HOLD_SECONDS}; wait: 0 to ${MAX_WAIT_SECONDS}.`
: `Seconds. wait: 0 to ${MAX_WAIT_SECONDS}; this does not extend a key tap.`,
}),
screenIndex: optionalNonNegativeIntegerSchema(),
frameId: Type.Optional(
Type.String({
description:
"Desktop coordinate actions: exact frame id returned by the most recent screenshot result.",
}),
),
windowRef: Type.Optional(
Type.String({
description:
"Opaque window reference for window actions; not valid for screenshot or wait.",
}),
),
browserRef: Type.Optional(
Type.String({ description: "Opaque browser reference from get_browser_state." }),
),
pageRef: Type.Optional(
Type.String({ description: "Opaque browser page reference from get_browser_state." }),
),
elementRef: Type.Optional(
Type.String({ description: "Opaque accessibility element reference from observation." }),
),
observationId: Type.Optional(
Type.String({
description:
"Window/browser input: observation id from the latest targeted observation; a desktop frameId cannot replace it.",
}),
),
deliveryMode: optionalStringEnum(["background", "foreground"] as const, {
description:
"Window-targeted input delivery. This does not turn desktop input into background window input.",
}),
query: Type.Optional(
Type.String({
description:
`${accessibilityTarget}: text filter.` +
(actions.includes("get_browser_state")
? " get_browser_state: requires snapshotFormat=semantic_v2 with browserRef and pageRef."
: ""),
}),
),
depth: Type.Optional(
Type.Integer({
minimum: 0,
maximum: 64,
description: `${accessibilityTarget}: maximum tree depth.`,
}),
),
maxElements: Type.Optional(
Type.Integer({
minimum: 1,
maximum: 2_000,
description: `${accessibilityTarget}: maximum returned elements.`,
}),
),
app: Type.Optional(Type.String()),
value: Type.Optional(Type.String()),
path: Type.Optional(
Type.Array(Type.String({ minLength: 1, maxLength: 200 }), { minItems: 1, maxItems: 16 }),
),
x1: Type.Optional(Type.Number({ minimum: 0 })),
y1: Type.Optional(Type.Number({ minimum: 0 })),
x2: Type.Optional(Type.Number({ minimum: 0 })),
y2: Type.Optional(Type.Number({ minimum: 0 })),
reason: optionalStringEnum([
"ax_tree_pixel_mismatch",
"background_delivery_failed",
"foreground_ineffective",
"no_window_target",
"other",
] as const),
snapshotFormat: optionalStringEnum(["dom_refs_v1", "semantic_v2"] as const),
continuation: Type.Optional(Type.String()),
includeScreenshot: Type.Optional(Type.Boolean()),
profile: optionalStringEnum(["isolated_new", "isolated_named"] as const),
profileName: Type.Optional(Type.String({ minLength: 1, maxLength: 64 })),
url: Type.Optional(Type.String()),
inputRoute: optionalStringEnum(["trusted", "dom_event"] as const),
mode: optionalStringEnum(["insert_text", "keystrokes"] as const),
replace: Type.Optional(Type.Boolean()),
dialogAction: optionalStringEnum(["inspect", "accept", "dismiss"] as const),
dialogRef: Type.Optional(Type.String()),
promptText: Type.Optional(Type.String()),
resourceHandle: Type.Optional(
Type.String({ description: "Opaque host-owned Computer Use resource handle." }),
),
resourceHandles: Type.Optional(
Type.Array(Type.String({ minLength: 1 }), { minItems: 1, maxItems: 32 }),
),
recordVideo: Type.Optional(Type.Boolean()),
delayMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 10_000 })),
stopOnError: Type.Optional(Type.Boolean()),
pointerAction: optionalStringEnum([
"hover",
"right_click",
"double_click",
"scroll",
"drag",
] as const),
destinationElementRef: Type.Optional(Type.String()),
deltaX: Type.Optional(Type.Number()),
deltaY: Type.Optional(Type.Number()),
});
}
|