File size: 3,246 Bytes
eb3f11e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | import type {
ComputerUseCapabilityDescriptor,
ComputerUseV2ActionName,
} from "../../plugins/computer-use-contract.js";
export const COMPUTER_ACT_COMMAND = "computer.act";
export const SCREEN_SNAPSHOT_COMMAND = "screen.snapshot";
export const COMPUTER_REF_WIDTH = 1280;
export const SCREENSHOT_QUALITY = 0.85;
export const AFTER_ACTION_SCREENSHOT_DELAY_MS = 500;
export const MAX_WAIT_SECONDS = 100;
export const MAX_HOLD_SECONDS = 10;
export const MODEL_OBSERVATION_MAX_ELEMENTS = 200;
export type ComputerToolAction = ComputerUseV2ActionName;
const COMPUTER_OBSERVATION_ACTIONS = new Set<string>([
"screenshot",
"wait",
"list_apps",
"list_windows",
"get_accessibility_tree",
"get_cursor_position",
"get_window_state",
"zoom",
"get_browser_state",
"get_recording_state",
] satisfies ComputerToolAction[]);
// Observations may refresh frame/ref caches, but do not perform setup or input.
// Keep result handling and replay classification on the same action contract.
export function isComputerObservationAction(
action: string | undefined,
dialogAction?: unknown,
): boolean {
return (
action !== undefined &&
(COMPUTER_OBSERVATION_ACTIONS.has(action) ||
(action === "browser_dialog" && dialogAction === "inspect"))
);
}
/** The owner binds the execution target and revalidates its authority before every dispatch. */
export type ComputerToolTransport = {
readonly computerUse?: ComputerUseCapabilityDescriptor;
resolveNode: (
query?: string,
signal?: AbortSignal,
) => Promise<{ nodeId: string; computerUse?: ComputerUseCapabilityDescriptor }>;
invoke: (params: {
nodeId: string;
command: typeof COMPUTER_ACT_COMMAND | typeof SCREEN_SNAPSHOT_COMMAND;
commandParams: Record<string, unknown>;
timeoutMs?: number;
idempotencyKey?: string;
signal?: AbortSignal;
}) => Promise<unknown>;
};
export type ComputerHost = { host: "gateway" } | { host: "node"; nodeId: string };
export type ComputerTarget = ComputerHost & { screenIndex: number };
export function computerHostKey(target: ComputerHost): string {
return target.host === "gateway" ? "gateway" : `node:${target.nodeId}`;
}
export function computerTargetDetails(target: ComputerHost) {
return target.host === "gateway" ? { target: "gateway" as const } : { node: target.nodeId };
}
export type ComputerFrame = {
target: ComputerTarget;
id: string;
displayFrameId: string;
contextEpoch: number;
};
export type ScreenshotCapture = {
base64: string;
displayFrameId: string;
mimeType: string;
width?: number;
height?: number;
};
export type ComputerObservationState = {
targetKey: string;
providerGeneration: string;
observationId: string;
imageCoordinates?:
| { kind: "unavailable" }
| { kind: "available"; scaleX: number; scaleY: number };
};
export type ComputerContextEpoch = {
value: number;
/** Tool result whose screenshot currently authorizes coordinates. */
frameToolCallId?: string;
/** Digest of the exact sanitized image the model received for that result. */
frameImageIdentity?: string;
};
export type ResolvedComputerTarget = {
target: ComputerTarget;
frame?: ComputerFrame;
capabilities?: ComputerUseCapabilityDescriptor;
};
|