gemini-cli / packages /core /src /agents /browser /analyzeScreenshot.test.ts

Add files using upload-large-folder tool

7a1ad33 verified about 1 month ago

9.19 kB

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import { describe, it, expect, vi, beforeEach } from 'vitest';
import { createAnalyzeScreenshotTool } from './analyzeScreenshot.js';
import type { BrowserManager, McpToolCallResult } from './browserManager.js';
import type { Config } from '../../config/config.js';
import type { MessageBus } from '../../confirmation-bus/message-bus.js';
import { Environment } from '@google/genai';

const mockMessageBus = {
  waitForConfirmation: vi.fn().mockResolvedValue({ approved: true }),
} as unknown as MessageBus;

function createMockBrowserManager(
  callToolResult?: McpToolCallResult,
): BrowserManager {
  return {
    callTool: vi.fn().mockResolvedValue(
      callToolResult ?? {
        content: [
          { type: 'text', text: 'Screenshot captured' },
          {
            type: 'image',
            data: 'base64encodeddata',
            mimeType: 'image/png',
          },
        ],
      },
    ),
  } as unknown as BrowserManager;
}

function createMockConfig(
  generateContentResult?: unknown,
  generateContentError?: Error,
  modelName: string = 'gemini-2.5-computer-use-preview-10-2025',
): Config {
  const generateContent = generateContentError
    ? vi.fn().mockRejectedValue(generateContentError)
    : vi.fn().mockResolvedValue(
        generateContentResult ?? {
          candidates: [
            {
              content: {
                parts: [
                  {
                    text: 'The blue submit button is at coordinates (250, 400).',
                  },
                ],
              },
            },
          ],
        },
      );

  return {
    getBrowserAgentConfig: vi.fn().mockReturnValue({
      customConfig: { visualModel: modelName },
    }),
    getContentGenerator: vi.fn().mockReturnValue({
      generateContent,
    }),
  } as unknown as Config;
}

describe('analyzeScreenshot', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  describe('createAnalyzeScreenshotTool', () => {
    it('creates a tool with the correct name and schema', () => {
      const browserManager = createMockBrowserManager();
      const config = createMockConfig();
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      expect(tool.name).toBe('analyze_screenshot');
    });
  });

  describe('AnalyzeScreenshotInvocation', () => {
    it('captures a screenshot and returns visual analysis', async () => {
      const browserManager = createMockBrowserManager();
      const config = createMockConfig();
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      const invocation = tool.build({
        instruction: 'Find the blue submit button',
      });
      const result = await invocation.execute({
        abortSignal: new AbortController().signal,
      });

      // Verify screenshot was captured
      expect(browserManager.callTool).toHaveBeenCalledWith(
        'take_screenshot',
        {},
      );

      // Verify the visual model was called
      const contentGenerator = config.getContentGenerator();
      expect(contentGenerator.generateContent).toHaveBeenCalledWith(
        expect.objectContaining({
          model: 'gemini-2.5-computer-use-preview-10-2025',
          config: expect.objectContaining({
            tools: [
              {
                computerUse: {
                  environment: Environment.ENVIRONMENT_BROWSER,
                  excludedPredefinedFunctions: [
                    'open_web_browser',
                    'click_at',
                    'key_combination',
                    'drag_and_drop',
                  ],
                },
              },
            ],
          }),
          contents: expect.arrayContaining([
            expect.objectContaining({
              role: 'user',
              parts: expect.arrayContaining([
                expect.objectContaining({
                  inlineData: {
                    mimeType: 'image/png',
                    data: 'base64encodeddata',
                  },
                }),
              ]),
            }),
          ]),
        }),
        'visual-analysis',
        'utility_tool',
      );

      // Verify result
      expect(result.llmContent).toContain('Visual Analysis Result');
      expect(result.llmContent).toContain(
        'The blue submit button is at coordinates (250, 400).',
      );
      expect(result.error).toBeUndefined();
    });

    it('omits computerUse tools for non-computer-use models', async () => {
      const browserManager = createMockBrowserManager();
      const config = createMockConfig(undefined, undefined, 'gemini-2.0-flash');
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      const invocation = tool.build({
        instruction: 'Find the search bar',
      });
      await invocation.execute({ abortSignal: new AbortController().signal });

      const contentGenerator = config.getContentGenerator();
      expect(contentGenerator.generateContent).toHaveBeenCalledWith(
        expect.objectContaining({
          model: 'gemini-2.0-flash',
          config: expect.not.objectContaining({
            tools: expect.anything(),
          }),
        }),
        'visual-analysis',
        'utility_tool',
      );
    });

    it('returns an error when screenshot capture fails (no image)', async () => {
      const browserManager = createMockBrowserManager({
        content: [{ type: 'text', text: 'No screenshot available' }],
      });
      const config = createMockConfig();
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      const invocation = tool.build({
        instruction: 'Find the button',
      });
      const result = await invocation.execute({
        abortSignal: new AbortController().signal,
      });

      expect(result.error).toBeDefined();
      expect(result.llmContent).toContain('Failed to capture screenshot');
      // Should NOT call the visual model
      const contentGenerator = config.getContentGenerator();
      expect(contentGenerator.generateContent).not.toHaveBeenCalled();
    });

    it('returns an error when visual model returns empty response', async () => {
      const browserManager = createMockBrowserManager();
      const config = createMockConfig({
        candidates: [{ content: { parts: [] } }],
      });
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      const invocation = tool.build({
        instruction: 'Check the layout',
      });
      const result = await invocation.execute({
        abortSignal: new AbortController().signal,
      });

      expect(result.error).toBeDefined();
      expect(result.llmContent).toContain('Visual model returned no analysis');
    });

    it('returns a model-unavailability fallback for 404 errors', async () => {
      const browserManager = createMockBrowserManager();
      const config = createMockConfig(
        undefined,
        new Error('Model not found: 404'),
      );
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      const invocation = tool.build({
        instruction: 'Find the red error',
      });
      const result = await invocation.execute({
        abortSignal: new AbortController().signal,
      });

      expect(result.error).toBeDefined();
      expect(result.llmContent).toContain(
        'Visual analysis model is not available',
      );
    });

    it('returns a model-unavailability fallback for 403 errors', async () => {
      const browserManager = createMockBrowserManager();
      const config = createMockConfig(
        undefined,
        new Error('permission denied: 403'),
      );
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      const invocation = tool.build({
        instruction: 'Identify the element',
      });
      const result = await invocation.execute({
        abortSignal: new AbortController().signal,
      });

      expect(result.error).toBeDefined();
      expect(result.llmContent).toContain(
        'Visual analysis model is not available',
      );
    });

    it('returns a generic error for non-model errors', async () => {
      const browserManager = createMockBrowserManager();
      const config = createMockConfig(undefined, new Error('Network timeout'));
      const tool = createAnalyzeScreenshotTool(
        browserManager,
        config,
        mockMessageBus,
      );

      const invocation = tool.build({
        instruction: 'Find something',
      });
      const result = await invocation.execute({
        abortSignal: new AbortController().signal,
      });

      expect(result.error).toBeDefined();
      expect(result.llmContent).toContain('Visual analysis failed');
      expect(result.llmContent).toContain('Network timeout');
    });
  });
});

	/**
	* @license
	* Copyright 2026 Google LLC
	* SPDX-License-Identifier: Apache-2.0
	*/

	import { describe, it, expect, vi, beforeEach } from 'vitest';
	import { createAnalyzeScreenshotTool } from './analyzeScreenshot.js';
	import type { BrowserManager, McpToolCallResult } from './browserManager.js';
	import type { Config } from '../../config/config.js';
	import type { MessageBus } from '../../confirmation-bus/message-bus.js';
	import { Environment } from '@google/genai';

	const mockMessageBus = {
	waitForConfirmation: vi.fn().mockResolvedValue({ approved: true }),
	} as unknown as MessageBus;

	function createMockBrowserManager(
	callToolResult?: McpToolCallResult,
	): BrowserManager {
	return {
	callTool: vi.fn().mockResolvedValue(
	callToolResult ?? {
	content: [
	{ type: 'text', text: 'Screenshot captured' },
	{
	type: 'image',
	data: 'base64encodeddata',
	mimeType: 'image/png',
	},
	],
	},
	),
	} as unknown as BrowserManager;
	}

	function createMockConfig(
	generateContentResult?: unknown,
	generateContentError?: Error,
	modelName: string = 'gemini-2.5-computer-use-preview-10-2025',
	): Config {
	const generateContent = generateContentError
	? vi.fn().mockRejectedValue(generateContentError)
	: vi.fn().mockResolvedValue(
	generateContentResult ?? {
	candidates: [
	{
	content: {
	parts: [
	{
	text: 'The blue submit button is at coordinates (250, 400).',
	},
	],
	},
	},
	],
	},
	);

	return {
	getBrowserAgentConfig: vi.fn().mockReturnValue({
	customConfig: { visualModel: modelName },
	}),
	getContentGenerator: vi.fn().mockReturnValue({
	generateContent,
	}),
	} as unknown as Config;
	}

	describe('analyzeScreenshot', () => {
	beforeEach(() => {
	vi.clearAllMocks();
	});

	describe('createAnalyzeScreenshotTool', () => {
	it('creates a tool with the correct name and schema', () => {
	const browserManager = createMockBrowserManager();
	const config = createMockConfig();
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	expect(tool.name).toBe('analyze_screenshot');
	});
	});

	describe('AnalyzeScreenshotInvocation', () => {
	it('captures a screenshot and returns visual analysis', async () => {
	const browserManager = createMockBrowserManager();
	const config = createMockConfig();
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	const invocation = tool.build({
	instruction: 'Find the blue submit button',
	});
	const result = await invocation.execute({
	abortSignal: new AbortController().signal,
	});

	// Verify screenshot was captured
	expect(browserManager.callTool).toHaveBeenCalledWith(
	'take_screenshot',
	{},
	);

	// Verify the visual model was called
	const contentGenerator = config.getContentGenerator();
	expect(contentGenerator.generateContent).toHaveBeenCalledWith(
	expect.objectContaining({
	model: 'gemini-2.5-computer-use-preview-10-2025',
	config: expect.objectContaining({
	tools: [
	{
	computerUse: {
	environment: Environment.ENVIRONMENT_BROWSER,
	excludedPredefinedFunctions: [
	'open_web_browser',
	'click_at',
	'key_combination',
	'drag_and_drop',
	],
	},
	},
	],
	}),
	contents: expect.arrayContaining([
	expect.objectContaining({
	role: 'user',
	parts: expect.arrayContaining([
	expect.objectContaining({
	inlineData: {
	mimeType: 'image/png',
	data: 'base64encodeddata',
	},
	}),
	]),
	}),
	]),
	}),
	'visual-analysis',
	'utility_tool',
	);

	// Verify result
	expect(result.llmContent).toContain('Visual Analysis Result');
	expect(result.llmContent).toContain(
	'The blue submit button is at coordinates (250, 400).',
	);
	expect(result.error).toBeUndefined();
	});

	it('omits computerUse tools for non-computer-use models', async () => {
	const browserManager = createMockBrowserManager();
	const config = createMockConfig(undefined, undefined, 'gemini-2.0-flash');
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	const invocation = tool.build({
	instruction: 'Find the search bar',
	});
	await invocation.execute({ abortSignal: new AbortController().signal });

	const contentGenerator = config.getContentGenerator();
	expect(contentGenerator.generateContent).toHaveBeenCalledWith(
	expect.objectContaining({
	model: 'gemini-2.0-flash',
	config: expect.not.objectContaining({
	tools: expect.anything(),
	}),
	}),
	'visual-analysis',
	'utility_tool',
	);
	});

	it('returns an error when screenshot capture fails (no image)', async () => {
	const browserManager = createMockBrowserManager({
	content: [{ type: 'text', text: 'No screenshot available' }],
	});
	const config = createMockConfig();
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	const invocation = tool.build({
	instruction: 'Find the button',
	});
	const result = await invocation.execute({
	abortSignal: new AbortController().signal,
	});

	expect(result.error).toBeDefined();
	expect(result.llmContent).toContain('Failed to capture screenshot');
	// Should NOT call the visual model
	const contentGenerator = config.getContentGenerator();
	expect(contentGenerator.generateContent).not.toHaveBeenCalled();
	});

	it('returns an error when visual model returns empty response', async () => {
	const browserManager = createMockBrowserManager();
	const config = createMockConfig({
	candidates: [{ content: { parts: [] } }],
	});
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	const invocation = tool.build({
	instruction: 'Check the layout',
	});
	const result = await invocation.execute({
	abortSignal: new AbortController().signal,
	});

	expect(result.error).toBeDefined();
	expect(result.llmContent).toContain('Visual model returned no analysis');
	});

	it('returns a model-unavailability fallback for 404 errors', async () => {
	const browserManager = createMockBrowserManager();
	const config = createMockConfig(
	undefined,
	new Error('Model not found: 404'),
	);
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	const invocation = tool.build({
	instruction: 'Find the red error',
	});
	const result = await invocation.execute({
	abortSignal: new AbortController().signal,
	});

	expect(result.error).toBeDefined();
	expect(result.llmContent).toContain(
	'Visual analysis model is not available',
	);
	});

	it('returns a model-unavailability fallback for 403 errors', async () => {
	const browserManager = createMockBrowserManager();
	const config = createMockConfig(
	undefined,
	new Error('permission denied: 403'),
	);
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	const invocation = tool.build({
	instruction: 'Identify the element',
	});
	const result = await invocation.execute({
	abortSignal: new AbortController().signal,
	});

	expect(result.error).toBeDefined();
	expect(result.llmContent).toContain(
	'Visual analysis model is not available',
	);
	});

	it('returns a generic error for non-model errors', async () => {
	const browserManager = createMockBrowserManager();
	const config = createMockConfig(undefined, new Error('Network timeout'));
	const tool = createAnalyzeScreenshotTool(
	browserManager,
	config,
	mockMessageBus,
	);

	const invocation = tool.build({
	instruction: 'Find something',
	});
	const result = await invocation.execute({
	abortSignal: new AbortController().signal,
	});

	expect(result.error).toBeDefined();
	expect(result.llmContent).toContain('Visual analysis failed');
	expect(result.llmContent).toContain('Network timeout');
	});
	});
	});