openclaw / src /agents /tools /computer-tool-result.ts

Add files using upload-large-folder tool

eb3f11e verified about 2 months ago

9.41 kB

import crypto from "node:crypto";
import { imageMimeFromFormat } from "@openclaw/media-core/mime";
import { readImageMetadataFromHeader } from "../../media/image-ops.js";
import type { ComputerActResult } from "../../plugins/computer-use-contract.js";
import { DEFAULT_IMAGE_MAX_DIMENSION_PX } from "../image-sanitization.js";
import type { AgentMessage, AgentToolResult } from "../runtime/index.js";
import { sanitizeContentBlocksImages } from "../tool-images.js";
import type {
  ComputerContextEpoch,
  ComputerObservationState,
  ComputerTarget,
  ComputerToolAction,
  ScreenshotCapture,
} from "./computer-tool-shared.js";
import {
  COMPUTER_REF_WIDTH,
  MODEL_OBSERVATION_MAX_ELEMENTS,
  computerTargetDetails,
} from "./computer-tool-shared.js";

type ModelObservationProjection = NonNullable<ComputerActResult["observation"]> & {
  truncatedElements?: number;
};

function projectComputerActResultMetadata(result: ComputerActResult) {
  let observation: ModelObservationProjection | undefined = result.observation
    ? { ...result.observation, ...(result.observation.base64 ? { base64: "[image]" } : {}) }
    : undefined;
  if (observation?.elements && observation.elements.length > MODEL_OBSERVATION_MAX_ELEMENTS) {
    observation = {
      ...observation,
      elements: observation.elements.slice(0, MODEL_OBSERVATION_MAX_ELEMENTS),
      truncatedElements: observation.elements.length - MODEL_OBSERVATION_MAX_ELEMENTS,
    };
  }
  const details = result.details ? { ...result.details } : undefined;
  if (
    details &&
    Array.isArray(details.elements) &&
    details.elements.length > MODEL_OBSERVATION_MAX_ELEMENTS
  ) {
    const originalLength = details.elements.length;
    details.elements = details.elements.slice(0, MODEL_OBSERVATION_MAX_ELEMENTS);
    details.truncatedElements = originalLength - MODEL_OBSERVATION_MAX_ELEMENTS;
  }
  return {
    ...result,
    ...(observation ? { observation } : {}),
    ...(details ? { details } : {}),
  };
}

export function computerActResultText(
  action: ComputerToolAction,
  result: ComputerActResult,
): string {
  return JSON.stringify({ action, ...projectComputerActResultMetadata(result) });
}

function computerFrameImageIdentity(
  content: AgentToolResult<unknown>["content"],
): string | undefined {
  const [image, duplicate] = content.filter((block) => block.type === "image");
  if (!image || duplicate) {
    return undefined;
  }
  return crypto
    .createHash("sha256")
    .update(JSON.stringify([image.mimeType, image.data]))
    .digest("hex");
}

function invalidateComputerFrame(contextEpoch: ComputerContextEpoch): boolean {
  if (contextEpoch.frameToolCallId === undefined && contextEpoch.frameImageIdentity === undefined) {
    return false;
  }
  contextEpoch.value += 1;
  delete contextEpoch.frameToolCallId;
  delete contextEpoch.frameImageIdentity;
  return true;
}

/**
 * Invalidate screenshot coordinates when the final model context no longer
 * contains the image produced by the tracked computer tool result.
 */
export function invalidateComputerFrameIfMissing(params: {
  contextEpoch: ComputerContextEpoch;
  messages: AgentMessage[];
  imagesBlocked?: boolean;
}): boolean {
  const frameToolCallId = params.contextEpoch.frameToolCallId;
  if (frameToolCallId === undefined || params.imagesBlocked) {
    return invalidateComputerFrame(params.contextEpoch);
  }

  let frameImageIdentity: string | undefined;
  for (let index = params.messages.length - 1; index >= 0; index -= 1) {
    const message = params.messages[index];
    if (
      message?.role !== "toolResult" ||
      message.toolName !== "computer" ||
      message.toolCallId !== frameToolCallId
    ) {
      continue;
    }
    frameImageIdentity = computerFrameImageIdentity(message.content);
    break;
  }

  if (
    frameImageIdentity !== undefined &&
    frameImageIdentity === params.contextEpoch.frameImageIdentity
  ) {
    return false;
  }
  return invalidateComputerFrame(params.contextEpoch);
}

/**
 * The reference frame width both the screenshot and the coordinates use.
 * Capped at the model's image sanitization limit so a persisted screenshot that
 * is replay-sanitized in a later turn is not resized underneath the coordinate
 * frame the model is still issuing `refWidth` against.
 */
export function resolveReferenceWidth(limits: { maxDimensionPx?: number }): number {
  const sanitizationLimit = limits.maxDimensionPx ?? DEFAULT_IMAGE_MAX_DIMENSION_PX;
  return Math.max(1, Math.min(COMPUTER_REF_WIDTH, sanitizationLimit));
}

async function projectComputerImage(params: {
  image?: { base64: string; mimeType: string };
  action: ComputerToolAction;
  referenceWidth: number;
  modelHasVision?: boolean;
}) {
  // Keep the delivered pixels within the replay cap so later turns cannot
  // resize the image underneath the coordinates bound to it.
  const content = await sanitizeContentBlocksImages(
    params.image && params.modelHasVision !== false
      ? [{ type: "image", data: params.image.base64, mimeType: params.image.mimeType }]
      : [],
    `computer:${params.action}`,
    { maxDimensionPx: params.referenceWidth },
  );
  const image = content.find((block) => block.type === "image");
  const dimensions = image ? readImageMetadataFromHeader(Buffer.from(image.data, "base64")) : null;
  return { content, image, dimensions };
}

export async function projectScreenshotResult(params: {
  capture: ScreenshotCapture;
  noteLines: string[];
  target: ComputerTarget;
  action: ComputerToolAction;
  referenceWidth: number;
  modelHasVision?: boolean;
}): Promise<{
  result: AgentToolResult<unknown>;
  frameId: string;
  imageIdentity?: string;
}> {
  const { capture, target } = params;
  const frameId = crypto.randomUUID();
  const { content, dimensions } = await projectComputerImage({ ...params, image: capture });
  const dims = dimensions ? `${dimensions.width}x${dimensions.height}` : "unknown size";
  const text = [
    ...params.noteLines,
    `screenshot ${dims} (screen ${target.screenIndex}, frameId ${frameId})`,
  ].join("\n");
  if (params.modelHasVision === false) {
    content.push({
      type: "text",
      text: "[model has no vision; screenshot omitted — use a vision-capable model for computer use]",
    });
  }
  const result = {
    content: [{ type: "text" as const, text }, ...content],
    details: {
      ...computerTargetDetails(target),
      action: params.action,
      width: dimensions?.width,
      height: dimensions?.height,
      screenIndex: target.screenIndex,
      frameId,
      refWidth: params.referenceWidth,
      media: { outbound: false },
    },
  };
  return { result, frameId, imageIdentity: computerFrameImageIdentity(result.content) };
}

export async function projectComputerActResult(params: {
  result: ComputerActResult;
  precedingAction?: { action: ComputerToolAction; result: ComputerActResult };
  target: ComputerTarget;
  action: ComputerToolAction;
  referenceWidth: number;
  modelHasVision?: boolean;
}): Promise<{
  result: AgentToolResult<unknown>;
  imageCoordinates?: ComputerObservationState["imageCoordinates"];
}> {
  const observation = params.result.observation;
  // Observation images have no context-presence tracking, so they must never be deduplicated.
  const { content, image, dimensions } = await projectComputerImage({
    ...params,
    image: observation?.base64
      ? {
          base64: observation.base64,
          mimeType: imageMimeFromFormat(observation.format ?? "png") ?? "image/png",
        }
      : undefined,
  });
  // Pixels belong only in image content. Metadata describes the delivered bitmap;
  // accessibility bounds retain the provider's coordinate space.
  const result = projectComputerActResultMetadata(params.result);
  if (result.observation && dimensions) {
    Object.assign(result.observation, dimensions, {
      format: image?.mimeType === "image/jpeg" ? "jpeg" : "png",
    });
  }
  let imageCoordinates: ComputerObservationState["imageCoordinates"];
  if (params.result.details?.coordinateSpace === "image-pixels") {
    imageCoordinates =
      dimensions && observation?.width && observation.height
        ? {
            kind: "available",
            scaleX: observation.width / dimensions.width,
            scaleY: observation.height / dimensions.height,
          }
        : { kind: "unavailable" };
  }
  return {
    result: {
      content: [
        ...(params.precedingAction
          ? [
              {
                type: "text" as const,
                text: computerActResultText(
                  params.precedingAction.action,
                  params.precedingAction.result,
                ),
              },
            ]
          : []),
        { type: "text", text: JSON.stringify({ action: params.action, ...result }) },
        ...content,
      ],
      details: {
        ...computerTargetDetails(params.target),
        action: params.precedingAction?.action ?? params.action,
        screenIndex: params.target.screenIndex,
        // Keep mutation evidence separate from the read's coordinate space and other metadata.
        result: params.precedingAction
          ? projectComputerActResultMetadata(params.precedingAction.result)
          : result,
        ...(params.precedingAction ? { followUpObservation: result } : {}),
        media: { outbound: false },
      },
    },
    imageCoordinates,
  };
}

	import crypto from "node:crypto";
	import { imageMimeFromFormat } from "@openclaw/media-core/mime";
	import { readImageMetadataFromHeader } from "../../media/image-ops.js";
	import type { ComputerActResult } from "../../plugins/computer-use-contract.js";
	import { DEFAULT_IMAGE_MAX_DIMENSION_PX } from "../image-sanitization.js";
	import type { AgentMessage, AgentToolResult } from "../runtime/index.js";
	import { sanitizeContentBlocksImages } from "../tool-images.js";
	import type {
	ComputerContextEpoch,
	ComputerObservationState,
	ComputerTarget,
	ComputerToolAction,
	ScreenshotCapture,
	} from "./computer-tool-shared.js";
	import {
	COMPUTER_REF_WIDTH,
	MODEL_OBSERVATION_MAX_ELEMENTS,
	computerTargetDetails,
	} from "./computer-tool-shared.js";

	type ModelObservationProjection = NonNullable<ComputerActResult["observation"]> & {
	truncatedElements?: number;
	};

	function projectComputerActResultMetadata(result: ComputerActResult) {
	let observation: ModelObservationProjection \| undefined = result.observation
	? { ...result.observation, ...(result.observation.base64 ? { base64: "[image]" } : {}) }
	: undefined;
	if (observation?.elements && observation.elements.length > MODEL_OBSERVATION_MAX_ELEMENTS) {
	observation = {
	...observation,
	elements: observation.elements.slice(0, MODEL_OBSERVATION_MAX_ELEMENTS),
	truncatedElements: observation.elements.length - MODEL_OBSERVATION_MAX_ELEMENTS,
	};
	}
	const details = result.details ? { ...result.details } : undefined;
	if (
	details &&
	Array.isArray(details.elements) &&
	details.elements.length > MODEL_OBSERVATION_MAX_ELEMENTS
	) {
	const originalLength = details.elements.length;
	details.elements = details.elements.slice(0, MODEL_OBSERVATION_MAX_ELEMENTS);
	details.truncatedElements = originalLength - MODEL_OBSERVATION_MAX_ELEMENTS;
	}
	return {
	...result,
	...(observation ? { observation } : {}),
	...(details ? { details } : {}),
	};
	}

	export function computerActResultText(
	action: ComputerToolAction,
	result: ComputerActResult,
	): string {
	return JSON.stringify({ action, ...projectComputerActResultMetadata(result) });
	}

	function computerFrameImageIdentity(
	content: AgentToolResult<unknown>["content"],
	): string \| undefined {
	const [image, duplicate] = content.filter((block) => block.type === "image");
	if (!image \|\| duplicate) {
	return undefined;
	}
	return crypto
	.createHash("sha256")
	.update(JSON.stringify([image.mimeType, image.data]))
	.digest("hex");
	}

	function invalidateComputerFrame(contextEpoch: ComputerContextEpoch): boolean {
	if (contextEpoch.frameToolCallId === undefined && contextEpoch.frameImageIdentity === undefined) {
	return false;
	}
	contextEpoch.value += 1;
	delete contextEpoch.frameToolCallId;
	delete contextEpoch.frameImageIdentity;
	return true;
	}

	/**
	* Invalidate screenshot coordinates when the final model context no longer
	* contains the image produced by the tracked computer tool result.
	*/
	export function invalidateComputerFrameIfMissing(params: {
	contextEpoch: ComputerContextEpoch;
	messages: AgentMessage[];
	imagesBlocked?: boolean;
	}): boolean {
	const frameToolCallId = params.contextEpoch.frameToolCallId;
	if (frameToolCallId === undefined \|\| params.imagesBlocked) {
	return invalidateComputerFrame(params.contextEpoch);
	}

	let frameImageIdentity: string \| undefined;
	for (let index = params.messages.length - 1; index >= 0; index -= 1) {
	const message = params.messages[index];
	if (
	message?.role !== "toolResult" \|\|
	message.toolName !== "computer" \|\|
	message.toolCallId !== frameToolCallId
	) {
	continue;
	}
	frameImageIdentity = computerFrameImageIdentity(message.content);
	break;
	}

	if (
	frameImageIdentity !== undefined &&
	frameImageIdentity === params.contextEpoch.frameImageIdentity
	) {
	return false;
	}
	return invalidateComputerFrame(params.contextEpoch);
	}

	/**
	* The reference frame width both the screenshot and the coordinates use.
	* Capped at the model's image sanitization limit so a persisted screenshot that
	* is replay-sanitized in a later turn is not resized underneath the coordinate
	* frame the model is still issuing `refWidth` against.
	*/
	export function resolveReferenceWidth(limits: { maxDimensionPx?: number }): number {
	const sanitizationLimit = limits.maxDimensionPx ?? DEFAULT_IMAGE_MAX_DIMENSION_PX;
	return Math.max(1, Math.min(COMPUTER_REF_WIDTH, sanitizationLimit));
	}

	async function projectComputerImage(params: {
	image?: { base64: string; mimeType: string };
	action: ComputerToolAction;
	referenceWidth: number;
	modelHasVision?: boolean;
	}) {
	// Keep the delivered pixels within the replay cap so later turns cannot
	// resize the image underneath the coordinates bound to it.
	const content = await sanitizeContentBlocksImages(
	params.image && params.modelHasVision !== false
	? [{ type: "image", data: params.image.base64, mimeType: params.image.mimeType }]
	: [],
	`computer:${params.action}`,
	{ maxDimensionPx: params.referenceWidth },
	);
	const image = content.find((block) => block.type === "image");
	const dimensions = image ? readImageMetadataFromHeader(Buffer.from(image.data, "base64")) : null;
	return { content, image, dimensions };
	}

	export async function projectScreenshotResult(params: {
	capture: ScreenshotCapture;
	noteLines: string[];
	target: ComputerTarget;
	action: ComputerToolAction;
	referenceWidth: number;
	modelHasVision?: boolean;
	}): Promise<{
	result: AgentToolResult<unknown>;
	frameId: string;
	imageIdentity?: string;
	}> {
	const { capture, target } = params;
	const frameId = crypto.randomUUID();
	const { content, dimensions } = await projectComputerImage({ ...params, image: capture });
	const dims = dimensions ? `${dimensions.width}x${dimensions.height}` : "unknown size";
	const text = [
	...params.noteLines,
	`screenshot ${dims} (screen ${target.screenIndex}, frameId ${frameId})`,
	].join("\n");
	if (params.modelHasVision === false) {
	content.push({
	type: "text",
	text: "[model has no vision; screenshot omitted — use a vision-capable model for computer use]",
	});
	}
	const result = {
	content: [{ type: "text" as const, text }, ...content],
	details: {
	...computerTargetDetails(target),
	action: params.action,
	width: dimensions?.width,
	height: dimensions?.height,
	screenIndex: target.screenIndex,
	frameId,
	refWidth: params.referenceWidth,
	media: { outbound: false },
	},
	};
	return { result, frameId, imageIdentity: computerFrameImageIdentity(result.content) };
	}

	export async function projectComputerActResult(params: {
	result: ComputerActResult;
	precedingAction?: { action: ComputerToolAction; result: ComputerActResult };
	target: ComputerTarget;
	action: ComputerToolAction;
	referenceWidth: number;
	modelHasVision?: boolean;
	}): Promise<{
	result: AgentToolResult<unknown>;
	imageCoordinates?: ComputerObservationState["imageCoordinates"];
	}> {
	const observation = params.result.observation;
	// Observation images have no context-presence tracking, so they must never be deduplicated.
	const { content, image, dimensions } = await projectComputerImage({
	...params,
	image: observation?.base64
	? {
	base64: observation.base64,
	mimeType: imageMimeFromFormat(observation.format ?? "png") ?? "image/png",
	}
	: undefined,
	});
	// Pixels belong only in image content. Metadata describes the delivered bitmap;
	// accessibility bounds retain the provider's coordinate space.
	const result = projectComputerActResultMetadata(params.result);
	if (result.observation && dimensions) {
	Object.assign(result.observation, dimensions, {
	format: image?.mimeType === "image/jpeg" ? "jpeg" : "png",
	});
	}
	let imageCoordinates: ComputerObservationState["imageCoordinates"];
	if (params.result.details?.coordinateSpace === "image-pixels") {
	imageCoordinates =
	dimensions && observation?.width && observation.height
	? {
	kind: "available",
	scaleX: observation.width / dimensions.width,
	scaleY: observation.height / dimensions.height,
	}
	: { kind: "unavailable" };
	}
	return {
	result: {
	content: [
	...(params.precedingAction
	? [
	{
	type: "text" as const,
	text: computerActResultText(
	params.precedingAction.action,
	params.precedingAction.result,
	),
	},
	]
	: []),
	{ type: "text", text: JSON.stringify({ action: params.action, ...result }) },
	...content,
	],
	details: {
	...computerTargetDetails(params.target),
	action: params.precedingAction?.action ?? params.action,
	screenIndex: params.target.screenIndex,
	// Keep mutation evidence separate from the read's coordinate space and other metadata.
	result: params.precedingAction
	? projectComputerActResultMetadata(params.precedingAction.result)
	: result,
	...(params.precedingAction ? { followUpObservation: result } : {}),
	media: { outbound: false },
	},
	},
	imageCoordinates,
	};
	}