/** * @license * Copyright 2026 Google LLC * SPDX-License-Identifier: Apache-2.0 */ /** * @fileoverview Tool for visual identification via a single model call. * * The semantic browser agent uses this tool when it needs to identify * elements by visual attributes not present in the accessibility tree * (e.g., color, layout, precise coordinates). * * Unlike the semantic agent which works with the accessibility tree, * this tool sends a screenshot to a computer-use model for visual analysis. * It returns the model's analysis (coordinates, element descriptions) back * to the browser agent, which retains full control of subsequent actions. */ import { DeclarativeTool, BaseToolInvocation, Kind, type ToolResult, type ToolInvocation, type ExecuteOptions, } from '../../tools/tools.js'; import { Environment } from '@google/genai'; import type { MessageBus } from '../../confirmation-bus/message-bus.js'; import type { BrowserManager } from './browserManager.js'; import type { Config } from '../../config/config.js'; import { getVisualAgentModel, isComputerUseModel, } from './modelAvailability.js'; import { debugLogger } from '../../utils/debugLogger.js'; import { LlmRole } from '../../telemetry/llmRole.js'; /** * System prompt for the visual analysis model call. */ const VISUAL_SYSTEM_PROMPT = `You are a Visual Analysis Agent. You receive a screenshot of a browser page and an instruction. Your job is to ANALYZE the screenshot and provide precise information that a browser automation agent can act on. COORDINATE SYSTEM: - Coordinates are pixel-based relative to the viewport - (0,0) is top-left of the visible area - Estimate element positions from the screenshot RESPONSE FORMAT: - For coordinate identification: provide exact (x, y) pixel coordinates - For element identification: describe the element's visual location and appearance - For layout analysis: describe the spatial relationships between elements - Be concise and actionable — the browser agent will use your response to decide what action to take IMPORTANT: - You are NOT performing actions — you are only providing visual analysis - Include coordinates when possible so the caller can use click_at(x, y) - If the element is not visible in the screenshot, say so explicitly`; /** * Invocation for the analyze_screenshot tool. * Makes a single generateContent call with a screenshot. */ class AnalyzeScreenshotInvocation extends BaseToolInvocation< Record, ToolResult > { constructor( private readonly browserManager: BrowserManager, private readonly config: Config, params: Record, messageBus: MessageBus, ) { super(params, messageBus, 'analyze_screenshot', 'Analyze Screenshot'); } getDescription(): string { const instruction = String(this.params['instruction'] ?? ''); return `Visual analysis: "${instruction}"`; } async execute({ abortSignal: signal }: ExecuteOptions): Promise { try { const instruction = String(this.params['instruction'] ?? ''); debugLogger.log(`Visual analysis requested: ${instruction}`); // Capture screenshot via MCP tool const screenshotResult = await this.browserManager.callTool( 'take_screenshot', {}, ); // Extract base64 image data from MCP response. // Search ALL content items for image type — MCP returns [text, image] // where content[0] is a text description and content[1] is the actual PNG. let screenshotBase64 = ''; let mimeType = 'image/png'; if (screenshotResult.content && Array.isArray(screenshotResult.content)) { for (const item of screenshotResult.content) { if (item.type === 'image' && item.data) { screenshotBase64 = item.data; mimeType = item.mimeType ?? 'image/png'; break; } } } if (!screenshotBase64) { return { llmContent: 'Failed to capture screenshot for visual analysis. Use accessibility tree elements instead.', returnDisplay: 'Screenshot capture failed', error: { message: 'Screenshot capture failed' }, }; } // Make a single generateContent call with the visual model const visualModel = getVisualAgentModel(this.config); const contentGenerator = this.config.getContentGenerator(); // Computer-use models require the computerUse tool declaration in every // request. We exclude all predefined action functions so the model // provides text analysis rather than issuing actions. // Non-computer-use models (e.g., gemini-2.0-flash) do plain text // analysis natively and don't need this declaration. const tools = isComputerUseModel(visualModel) ? [ { computerUse: { environment: Environment.ENVIRONMENT_BROWSER, excludedPredefinedFunctions: [ 'open_web_browser', 'click_at', 'key_combination', 'drag_and_drop', ], }, }, ] : undefined; const response = await contentGenerator.generateContent( { model: visualModel, config: { temperature: 0, topP: 0.95, systemInstruction: VISUAL_SYSTEM_PROMPT, abortSignal: signal, ...(tools ? { tools } : {}), }, contents: [ { role: 'user', parts: [ { text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`, }, { inlineData: { mimeType, data: screenshotBase64, }, }, ], }, ], }, 'visual-analysis', LlmRole.UTILITY_TOOL, ); // Extract response content. Computer-use models may still return // functionCall parts even with exclusions, so we handle both text // and functionCall parts defensively. const parts = response.candidates?.[0]?.content?.parts ?? []; const textParts = parts.filter((p) => p.text).map((p) => p.text!); const functionCallParts = parts .filter((p) => p.functionCall) .map((p) => { const fc = p.functionCall!; const argsStr = fc.args ? JSON.stringify(fc.args) : ''; return `Action: ${fc.name}${argsStr ? ` with args ${argsStr}` : ''}`; }); const responseText = [...textParts, ...functionCallParts].join('\n'); if (!responseText) { return { llmContent: 'Visual model returned no analysis. Use accessibility tree elements instead.', returnDisplay: 'Visual analysis returned empty response', error: { message: 'Empty visual analysis response' }, }; } debugLogger.log(`Visual analysis complete: ${responseText}`); return { llmContent: `Visual Analysis Result:\n${responseText}`, returnDisplay: `Visual Analysis Result:\n${responseText}`, }; } catch (error) { const errorMsg = error instanceof Error ? error.message : String(error); debugLogger.error(`Visual analysis failed: ${errorMsg}`); // Provide a graceful fallback message for model unavailability const isModelError = errorMsg.includes('404') || errorMsg.includes('403') || errorMsg.includes('not found') || errorMsg.includes('permission'); const fallbackMsg = isModelError ? 'Visual analysis model is not available. Use accessibility tree elements (uids from take_snapshot) for all interactions instead.' : `Visual analysis failed: ${errorMsg}. Use accessibility tree elements instead.`; return { llmContent: fallbackMsg, returnDisplay: fallbackMsg, error: { message: errorMsg }, }; } } } /** * DeclarativeTool for screenshot-based visual analysis. */ class AnalyzeScreenshotTool extends DeclarativeTool< Record, ToolResult > { constructor( private readonly browserManager: BrowserManager, private readonly config: Config, messageBus: MessageBus, ) { super( 'analyze_screenshot', 'analyze_screenshot', 'Analyze the current page visually using a screenshot. Use when you need to identify elements by visual attributes (color, layout, position) not available in the accessibility tree, or when you need precise pixel coordinates for click_at. Returns visual analysis — you perform the actions yourself.', Kind.Other, { type: 'object', properties: { instruction: { type: 'string', description: 'What to identify or analyze visually (e.g., "Find the coordinates of the blue submit button", "What is the layout of the navigation menu?").', }, }, required: ['instruction'], }, messageBus, true, // isOutputMarkdown false, // canUpdateOutput ); } build( params: Record, ): ToolInvocation, ToolResult> { return new AnalyzeScreenshotInvocation( this.browserManager, this.config, params, this.messageBus, ); } } /** * Creates the analyze_screenshot tool for the browser agent. */ export function createAnalyzeScreenshotTool( browserManager: BrowserManager, config: Config, messageBus: MessageBus, ): AnalyzeScreenshotTool { return new AnalyzeScreenshotTool(browserManager, config, messageBus); }