gemini-cli / packages /core /src /agents /browser /analyzeScreenshot.ts

Add files using upload-large-folder tool

7a1ad33 verified about 1 month ago

9.81 kB

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

/**
 * @fileoverview Tool for visual identification via a single model call.
 *
 * The semantic browser agent uses this tool when it needs to identify
 * elements by visual attributes not present in the accessibility tree
 * (e.g., color, layout, precise coordinates).
 *
 * Unlike the semantic agent which works with the accessibility tree,
 * this tool sends a screenshot to a computer-use model for visual analysis.
 * It returns the model's analysis (coordinates, element descriptions) back
 * to the browser agent, which retains full control of subsequent actions.
 */

import {
  DeclarativeTool,
  BaseToolInvocation,
  Kind,
  type ToolResult,
  type ToolInvocation,
  type ExecuteOptions,
} from '../../tools/tools.js';
import { Environment } from '@google/genai';
import type { MessageBus } from '../../confirmation-bus/message-bus.js';
import type { BrowserManager } from './browserManager.js';
import type { Config } from '../../config/config.js';
import {
  getVisualAgentModel,
  isComputerUseModel,
} from './modelAvailability.js';
import { debugLogger } from '../../utils/debugLogger.js';
import { LlmRole } from '../../telemetry/llmRole.js';

/**
 * System prompt for the visual analysis model call.
 */
const VISUAL_SYSTEM_PROMPT = `You are a Visual Analysis Agent. You receive a screenshot of a browser page and an instruction.
Your job is to ANALYZE the screenshot and provide precise information that a browser automation agent can act on.
COORDINATE SYSTEM:
- Coordinates are pixel-based relative to the viewport
- (0,0) is top-left of the visible area
- Estimate element positions from the screenshot
RESPONSE FORMAT:
- For coordinate identification: provide exact (x, y) pixel coordinates
- For element identification: describe the element's visual location and appearance
- For layout analysis: describe the spatial relationships between elements
- Be concise and actionable — the browser agent will use your response to decide what action to take
IMPORTANT:
- You are NOT performing actions — you are only providing visual analysis
- Include coordinates when possible so the caller can use click_at(x, y)
- If the element is not visible in the screenshot, say so explicitly`;

/**
 * Invocation for the analyze_screenshot tool.
 * Makes a single generateContent call with a screenshot.
 */
class AnalyzeScreenshotInvocation extends BaseToolInvocation<
  Record<string, unknown>,
  ToolResult
> {
  constructor(
    private readonly browserManager: BrowserManager,
    private readonly config: Config,
    params: Record<string, unknown>,
    messageBus: MessageBus,
  ) {
    super(params, messageBus, 'analyze_screenshot', 'Analyze Screenshot');
  }

  getDescription(): string {
    const instruction = String(this.params['instruction'] ?? '');
    return `Visual analysis: "${instruction}"`;
  }

  async execute({ abortSignal: signal }: ExecuteOptions): Promise<ToolResult> {
    try {
      const instruction = String(this.params['instruction'] ?? '');

      debugLogger.log(`Visual analysis requested: ${instruction}`);

      // Capture screenshot via MCP tool
      const screenshotResult = await this.browserManager.callTool(
        'take_screenshot',
        {},
      );

      // Extract base64 image data from MCP response.
      // Search ALL content items for image type — MCP returns [text, image]
      // where content[0] is a text description and content[1] is the actual PNG.
      let screenshotBase64 = '';
      let mimeType = 'image/png';
      if (screenshotResult.content && Array.isArray(screenshotResult.content)) {
        for (const item of screenshotResult.content) {
          if (item.type === 'image' && item.data) {
            screenshotBase64 = item.data;
            mimeType = item.mimeType ?? 'image/png';
            break;
          }
        }
      }

      if (!screenshotBase64) {
        return {
          llmContent:
            'Failed to capture screenshot for visual analysis. Use accessibility tree elements instead.',
          returnDisplay: 'Screenshot capture failed',
          error: { message: 'Screenshot capture failed' },
        };
      }

      // Make a single generateContent call with the visual model
      const visualModel = getVisualAgentModel(this.config);
      const contentGenerator = this.config.getContentGenerator();

      // Computer-use models require the computerUse tool declaration in every
      // request. We exclude all predefined action functions so the model
      // provides text analysis rather than issuing actions.
      // Non-computer-use models (e.g., gemini-2.0-flash) do plain text
      // analysis natively and don't need this declaration.
      const tools = isComputerUseModel(visualModel)
        ? [
            {
              computerUse: {
                environment: Environment.ENVIRONMENT_BROWSER,
                excludedPredefinedFunctions: [
                  'open_web_browser',
                  'click_at',
                  'key_combination',
                  'drag_and_drop',
                ],
              },
            },
          ]
        : undefined;

      const response = await contentGenerator.generateContent(
        {
          model: visualModel,
          config: {
            temperature: 0,
            topP: 0.95,
            systemInstruction: VISUAL_SYSTEM_PROMPT,
            abortSignal: signal,
            ...(tools ? { tools } : {}),
          },
          contents: [
            {
              role: 'user',
              parts: [
                {
                  text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`,
                },
                {
                  inlineData: {
                    mimeType,
                    data: screenshotBase64,
                  },
                },
              ],
            },
          ],
        },
        'visual-analysis',
        LlmRole.UTILITY_TOOL,
      );

      // Extract response content. Computer-use models may still return
      // functionCall parts even with exclusions, so we handle both text
      // and functionCall parts defensively.
      const parts = response.candidates?.[0]?.content?.parts ?? [];

      const textParts = parts.filter((p) => p.text).map((p) => p.text!);

      const functionCallParts = parts
        .filter((p) => p.functionCall)
        .map((p) => {
          const fc = p.functionCall!;
          const argsStr = fc.args ? JSON.stringify(fc.args) : '';
          return `Action: ${fc.name}${argsStr ? ` with args ${argsStr}` : ''}`;
        });

      const responseText = [...textParts, ...functionCallParts].join('\n');

      if (!responseText) {
        return {
          llmContent:
            'Visual model returned no analysis. Use accessibility tree elements instead.',
          returnDisplay: 'Visual analysis returned empty response',
          error: { message: 'Empty visual analysis response' },
        };
      }

      debugLogger.log(`Visual analysis complete: ${responseText}`);

      return {
        llmContent: `Visual Analysis Result:\n${responseText}`,
        returnDisplay: `Visual Analysis Result:\n${responseText}`,
      };
    } catch (error) {
      const errorMsg = error instanceof Error ? error.message : String(error);
      debugLogger.error(`Visual analysis failed: ${errorMsg}`);

      // Provide a graceful fallback message for model unavailability
      const isModelError =
        errorMsg.includes('404') ||
        errorMsg.includes('403') ||
        errorMsg.includes('not found') ||
        errorMsg.includes('permission');

      const fallbackMsg = isModelError
        ? 'Visual analysis model is not available. Use accessibility tree elements (uids from take_snapshot) for all interactions instead.'
        : `Visual analysis failed: ${errorMsg}. Use accessibility tree elements instead.`;

      return {
        llmContent: fallbackMsg,
        returnDisplay: fallbackMsg,
        error: { message: errorMsg },
      };
    }
  }
}

/**
 * DeclarativeTool for screenshot-based visual analysis.
 */
class AnalyzeScreenshotTool extends DeclarativeTool<
  Record<string, unknown>,
  ToolResult
> {
  constructor(
    private readonly browserManager: BrowserManager,
    private readonly config: Config,
    messageBus: MessageBus,
  ) {
    super(
      'analyze_screenshot',
      'analyze_screenshot',
      'Analyze the current page visually using a screenshot. Use when you need to identify elements by visual attributes (color, layout, position) not available in the accessibility tree, or when you need precise pixel coordinates for click_at. Returns visual analysis — you perform the actions yourself.',
      Kind.Other,
      {
        type: 'object',
        properties: {
          instruction: {
            type: 'string',
            description:
              'What to identify or analyze visually (e.g., "Find the coordinates of the blue submit button", "What is the layout of the navigation menu?").',
          },
        },
        required: ['instruction'],
      },
      messageBus,
      true, // isOutputMarkdown
      false, // canUpdateOutput
    );
  }

  build(
    params: Record<string, unknown>,
  ): ToolInvocation<Record<string, unknown>, ToolResult> {
    return new AnalyzeScreenshotInvocation(
      this.browserManager,
      this.config,
      params,
      this.messageBus,
    );
  }
}

/**
 * Creates the analyze_screenshot tool for the browser agent.
 */
export function createAnalyzeScreenshotTool(
  browserManager: BrowserManager,
  config: Config,
  messageBus: MessageBus,
): AnalyzeScreenshotTool {
  return new AnalyzeScreenshotTool(browserManager, config, messageBus);
}

	/**
	* @license
	* Copyright 2026 Google LLC
	* SPDX-License-Identifier: Apache-2.0
	*/

	/**
	* @fileoverview Tool for visual identification via a single model call.
	*
	* The semantic browser agent uses this tool when it needs to identify
	* elements by visual attributes not present in the accessibility tree
	* (e.g., color, layout, precise coordinates).
	*
	* Unlike the semantic agent which works with the accessibility tree,
	* this tool sends a screenshot to a computer-use model for visual analysis.
	* It returns the model's analysis (coordinates, element descriptions) back
	* to the browser agent, which retains full control of subsequent actions.
	*/

	import {
	DeclarativeTool,
	BaseToolInvocation,
	Kind,
	type ToolResult,
	type ToolInvocation,
	type ExecuteOptions,
	} from '../../tools/tools.js';
	import { Environment } from '@google/genai';
	import type { MessageBus } from '../../confirmation-bus/message-bus.js';
	import type { BrowserManager } from './browserManager.js';
	import type { Config } from '../../config/config.js';
	import {
	getVisualAgentModel,
	isComputerUseModel,
	} from './modelAvailability.js';
	import { debugLogger } from '../../utils/debugLogger.js';
	import { LlmRole } from '../../telemetry/llmRole.js';

	/**
	* System prompt for the visual analysis model call.
	*/
	const VISUAL_SYSTEM_PROMPT = `You are a Visual Analysis Agent. You receive a screenshot of a browser page and an instruction.

	Your job is to ANALYZE the screenshot and provide precise information that a browser automation agent can act on.

	COORDINATE SYSTEM:
	- Coordinates are pixel-based relative to the viewport
	- (0,0) is top-left of the visible area
	- Estimate element positions from the screenshot

	RESPONSE FORMAT:
	- For coordinate identification: provide exact (x, y) pixel coordinates
	- For element identification: describe the element's visual location and appearance
	- For layout analysis: describe the spatial relationships between elements
	- Be concise and actionable — the browser agent will use your response to decide what action to take

	IMPORTANT:
	- You are NOT performing actions — you are only providing visual analysis
	- Include coordinates when possible so the caller can use click_at(x, y)
	- If the element is not visible in the screenshot, say so explicitly`;

	/**
	* Invocation for the analyze_screenshot tool.
	* Makes a single generateContent call with a screenshot.
	*/
	class AnalyzeScreenshotInvocation extends BaseToolInvocation<
	Record<string, unknown>,
	ToolResult
	> {
	constructor(
	private readonly browserManager: BrowserManager,
	private readonly config: Config,
	params: Record<string, unknown>,
	messageBus: MessageBus,
	) {
	super(params, messageBus, 'analyze_screenshot', 'Analyze Screenshot');
	}

	getDescription(): string {
	const instruction = String(this.params['instruction'] ?? '');
	return `Visual analysis: "${instruction}"`;
	}

	async execute({ abortSignal: signal }: ExecuteOptions): Promise<ToolResult> {
	try {
	const instruction = String(this.params['instruction'] ?? '');

	debugLogger.log(`Visual analysis requested: ${instruction}`);

	// Capture screenshot via MCP tool
	const screenshotResult = await this.browserManager.callTool(
	'take_screenshot',
	{},
	);

	// Extract base64 image data from MCP response.
	// Search ALL content items for image type — MCP returns [text, image]
	// where content[0] is a text description and content[1] is the actual PNG.
	let screenshotBase64 = '';
	let mimeType = 'image/png';
	if (screenshotResult.content && Array.isArray(screenshotResult.content)) {
	for (const item of screenshotResult.content) {
	if (item.type === 'image' && item.data) {
	screenshotBase64 = item.data;
	mimeType = item.mimeType ?? 'image/png';
	break;
	}
	}
	}

	if (!screenshotBase64) {
	return {
	llmContent:
	'Failed to capture screenshot for visual analysis. Use accessibility tree elements instead.',
	returnDisplay: 'Screenshot capture failed',
	error: { message: 'Screenshot capture failed' },
	};
	}

	// Make a single generateContent call with the visual model
	const visualModel = getVisualAgentModel(this.config);
	const contentGenerator = this.config.getContentGenerator();

	// Computer-use models require the computerUse tool declaration in every
	// request. We exclude all predefined action functions so the model
	// provides text analysis rather than issuing actions.
	// Non-computer-use models (e.g., gemini-2.0-flash) do plain text
	// analysis natively and don't need this declaration.
	const tools = isComputerUseModel(visualModel)
	? [
	{
	computerUse: {
	environment: Environment.ENVIRONMENT_BROWSER,
	excludedPredefinedFunctions: [
	'open_web_browser',
	'click_at',
	'key_combination',
	'drag_and_drop',
	],
	},
	},
	]
	: undefined;

	const response = await contentGenerator.generateContent(
	{
	model: visualModel,
	config: {
	temperature: 0,
	topP: 0.95,
	systemInstruction: VISUAL_SYSTEM_PROMPT,
	abortSignal: signal,
	...(tools ? { tools } : {}),
	},
	contents: [
	{
	role: 'user',
	parts: [
	{
	text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`,
	},
	{
	inlineData: {
	mimeType,
	data: screenshotBase64,
	},
	},
	],
	},
	],
	},
	'visual-analysis',
	LlmRole.UTILITY_TOOL,
	);

	// Extract response content. Computer-use models may still return
	// functionCall parts even with exclusions, so we handle both text
	// and functionCall parts defensively.
	const parts = response.candidates?.[0]?.content?.parts ?? [];

	const textParts = parts.filter((p) => p.text).map((p) => p.text!);

	const functionCallParts = parts
	.filter((p) => p.functionCall)
	.map((p) => {
	const fc = p.functionCall!;
	const argsStr = fc.args ? JSON.stringify(fc.args) : '';
	return `Action: ${fc.name}${argsStr ? ` with args ${argsStr}` : ''}`;
	});

	const responseText = [...textParts, ...functionCallParts].join('\n');

	if (!responseText) {
	return {
	llmContent:
	'Visual model returned no analysis. Use accessibility tree elements instead.',
	returnDisplay: 'Visual analysis returned empty response',
	error: { message: 'Empty visual analysis response' },
	};
	}

	debugLogger.log(`Visual analysis complete: ${responseText}`);

	return {
	llmContent: `Visual Analysis Result:\n${responseText}`,
	returnDisplay: `Visual Analysis Result:\n${responseText}`,
	};
	} catch (error) {
	const errorMsg = error instanceof Error ? error.message : String(error);
	debugLogger.error(`Visual analysis failed: ${errorMsg}`);

	// Provide a graceful fallback message for model unavailability
	const isModelError =
	errorMsg.includes('404') \|\|
	errorMsg.includes('403') \|\|
	errorMsg.includes('not found') \|\|
	errorMsg.includes('permission');

	const fallbackMsg = isModelError
	? 'Visual analysis model is not available. Use accessibility tree elements (uids from take_snapshot) for all interactions instead.'
	: `Visual analysis failed: ${errorMsg}. Use accessibility tree elements instead.`;

	return {
	llmContent: fallbackMsg,
	returnDisplay: fallbackMsg,
	error: { message: errorMsg },
	};
	}
	}
	}

	/**
	* DeclarativeTool for screenshot-based visual analysis.
	*/
	class AnalyzeScreenshotTool extends DeclarativeTool<
	Record<string, unknown>,
	ToolResult
	> {
	constructor(
	private readonly browserManager: BrowserManager,
	private readonly config: Config,
	messageBus: MessageBus,
	) {
	super(
	'analyze_screenshot',
	'analyze_screenshot',
	'Analyze the current page visually using a screenshot. Use when you need to identify elements by visual attributes (color, layout, position) not available in the accessibility tree, or when you need precise pixel coordinates for click_at. Returns visual analysis — you perform the actions yourself.',
	Kind.Other,
	{
	type: 'object',
	properties: {
	instruction: {
	type: 'string',
	description:
	'What to identify or analyze visually (e.g., "Find the coordinates of the blue submit button", "What is the layout of the navigation menu?").',
	},
	},
	required: ['instruction'],
	},
	messageBus,
	true, // isOutputMarkdown
	false, // canUpdateOutput
	);
	}

	build(
	params: Record<string, unknown>,
	): ToolInvocation<Record<string, unknown>, ToolResult> {
	return new AnalyzeScreenshotInvocation(
	this.browserManager,
	this.config,
	params,
	this.messageBus,
	);
	}
	}

	/**
	* Creates the analyze_screenshot tool for the browser agent.
	*/
	export function createAnalyzeScreenshotTool(
	browserManager: BrowserManager,
	config: Config,
	messageBus: MessageBus,
	): AnalyzeScreenshotTool {
	return new AnalyzeScreenshotTool(browserManager, config, messageBus);
	}