File size: 6,900 Bytes
eb3f11e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | import crypto from "node:crypto";
import type { ComputerUseCapabilityDescriptor } from "../../plugins/computer-use-contract.js";
import {
type EligibleNodeMessages,
resolveEligibleNodeFromList,
} from "../../shared/node-resolve.js";
import { isEligibleComputerNode } from "../computer-use-node-capabilities.js";
import {
bindGatewayComputerCleanup,
loadGatewayComputerStatus,
type GatewayComputerStatus,
} from "./computer-tool-gateway.js";
import type { ComputerHost, ComputerToolTransport } from "./computer-tool-shared.js";
import { COMPUTER_ACT_COMMAND, SCREEN_SNAPSHOT_COMMAND } from "./computer-tool-shared.js";
import { getGatewayToolCallerIdentity } from "./gateway-caller-context.js";
import {
callGatewayTool,
shouldUseInProcessGatewayTool,
type GatewayCallOptions,
} from "./gateway.js";
import { listNodes, type NodeListNode } from "./nodes-utils.js";
export type ComputerBinding = {
host: ComputerHost;
gatewayOpts: GatewayCallOptions;
capabilities?: ComputerUseCapabilityDescriptor;
invoke: (
request: Omit<Parameters<ComputerToolTransport["invoke"]>[0], "nodeId">,
) => Promise<unknown>;
};
export const NOT_COMPUTER_CAPABLE_HINT =
"enable Computer Control in the OpenClaw app and approve the pairing update";
const COMPUTER_NODE_MESSAGES: EligibleNodeMessages<NodeListNode> = {
ineligibleExact: (query, eligibleIds) =>
`node "${query}" is not computer-capable (needs a connected node advertising ${COMPUTER_ACT_COMMAND} and ${SCREEN_SNAPSHOT_COMMAND}; ${NOT_COMPUTER_CAPABLE_HINT}; ` +
`eligible node ids: ${eligibleIds})`,
nameResolveFailed: (reason, eligibleIds) =>
`${reason} (eligible computer-capable node ids: ${eligibleIds})`,
noneEligible: () =>
`no connected computer-capable node (a node must advertise ${COMPUTER_ACT_COMMAND} and ${SCREEN_SNAPSHOT_COMMAND}; ${NOT_COMPUTER_CAPABLE_HINT})`,
multipleEligible: (eligible) =>
`multiple computer-capable nodes connected; pass node explicitly: ${eligible
.map((node) => node.nodeId)
.join(", ")}`,
};
async function resolveComputerNode(
gatewayOpts: GatewayCallOptions,
query?: string,
signal?: AbortSignal,
): Promise<NodeListNode> {
const nodes = await listNodes(gatewayOpts, signal);
return resolveEligibleNodeFromList(nodes, query, isEligibleComputerNode, COMPUTER_NODE_MESSAGES);
}
async function invokeNodeCommand(params: {
gatewayOpts: GatewayCallOptions;
nodeId: string;
command: string;
commandParams: Record<string, unknown>;
timeoutMs?: number;
idempotencyKey?: string;
signal?: AbortSignal;
}): Promise<unknown> {
const raw = await callGatewayTool<{ payload: unknown }>(
"node.invoke",
params.gatewayOpts,
{
nodeId: params.nodeId,
command: params.command,
params: params.commandParams,
timeoutMs: params.timeoutMs,
idempotencyKey: params.idempotencyKey ?? crypto.randomUUID(),
},
{ signal: params.signal },
);
return raw && typeof raw === "object" && Object.hasOwn(raw, "payload") ? raw.payload : raw;
}
export async function resolveComputerBinding(params: {
executionId: string;
sessionTransport?: ComputerToolTransport;
gatewayStatus?: GatewayComputerStatus;
target?: "gateway" | "node";
node?: string;
gatewayOpts: GatewayCallOptions;
signal?: AbortSignal;
}): Promise<ComputerBinding> {
const sessionTransport = params.sessionTransport;
if (sessionTransport) {
const node = await sessionTransport.resolveNode(params.node, params.signal);
return {
host: { host: "node", nodeId: node.nodeId },
gatewayOpts: {},
capabilities: sessionTransport.computerUse ?? node.computerUse,
invoke: (request) => sessionTransport.invoke({ ...request, nodeId: node.nodeId }),
};
}
const gatewayOverride =
params.gatewayOpts.gatewayUrl !== undefined || params.gatewayOpts.gatewayToken !== undefined;
// Published remote Gateways select paired nodes without the newer computer RPCs.
if (
params.target !== "node" &&
params.node === undefined &&
(params.target === "gateway" || !gatewayOverride)
) {
const assertHostedCaller = () => {
if (
!shouldUseInProcessGatewayTool(params.gatewayOpts) ||
!getGatewayToolCallerIdentity()?.operationalRunInstance
) {
throw new Error(
"Gateway computer control requires an agent run hosted by that Gateway. " +
"Use a paired node, or use computer.invoke over one persistent operator RPC connection.",
);
}
};
if (params.target === "gateway") {
assertHostedCaller();
}
const prepared = !gatewayOverride ? params.gatewayStatus : undefined;
const usePrepared =
prepared?.available === true ||
(prepared?.configured === false && params.target !== "gateway");
const gateway =
(usePrepared ? prepared : undefined) ??
(await loadGatewayComputerStatus(params.gatewayOpts, params.signal));
if (gateway.available) {
assertHostedCaller();
const close = await bindGatewayComputerCleanup({
options: params.gatewayOpts,
generation: gateway.computerUse.provider.generation,
executionId: params.executionId,
});
return {
host: { host: "gateway" },
gatewayOpts: params.gatewayOpts,
capabilities: gateway.computerUse,
invoke: async (request) => {
if (
close &&
request.command === COMPUTER_ACT_COMMAND &&
request.commandParams.action === "__close_execution"
) {
return await close(
typeof request.commandParams.reason === "string"
? request.commandParams.reason
: "completed",
);
}
const raw = await callGatewayTool<{ payload: unknown }>(
"computer.invoke",
params.gatewayOpts,
{
generation: gateway.computerUse.provider.generation,
command: request.command,
params: request.commandParams,
timeoutMs: request.timeoutMs,
idempotencyKey: request.idempotencyKey ?? crypto.randomUUID(),
},
{ signal: request.signal },
);
return raw.payload;
},
};
}
if (gateway.configured || params.target === "gateway") {
throw new Error(
(gateway.configured ? gateway.error : undefined) ??
"Gateway computer is unavailable; enable its computer provider and desktop session",
);
}
}
const node = await resolveComputerNode(params.gatewayOpts, params.node, params.signal);
return {
host: { host: "node", nodeId: node.nodeId },
gatewayOpts: params.gatewayOpts,
capabilities: node.computerUse,
invoke: (request) =>
invokeNodeCommand({ ...request, nodeId: node.nodeId, gatewayOpts: params.gatewayOpts }),
};
}
|