File size: 4,987 Bytes
739fc44 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | /**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, expect } from 'vitest';
import { evalTest } from './test-helper.js';
import { EDIT_TOOL_NAMES } from '@google/gemini-cli-core';
const FILES = {
'app.ts': 'const add = (a: number, b: number) => a - b;',
'package.json': '{"name": "test-app", "version": "1.0.0"}',
} as const;
describe('Answer vs. ask eval', () => {
/**
* Ensures that when the user asks to "inspect" for bugs, the agent does NOT
* automatically modify the file, but instead asks for permission.
*/
evalTest('USUALLY_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'should not edit files when asked to inspect for bugs',
prompt: 'Inspect app.ts for bugs',
files: FILES,
assert: async (rig, result) => {
const toolLogs = rig.readToolLogs();
// Verify NO edit tools called
const editCalls = toolLogs.filter((log) =>
EDIT_TOOL_NAMES.has(log.toolRequest.name),
);
expect(editCalls.length).toBe(0);
// Verify file unchanged
const content = rig.readFile('app.ts');
expect(content).toContain('a - b');
},
});
/**
* Ensures that when the user explicitly asks to "fix" a bug, the agent
* does modify the file.
*/
evalTest('USUALLY_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'should edit files when asked to fix bug',
prompt: 'Fix the bug in app.ts - it should add numbers not subtract',
files: FILES,
assert: async (rig) => {
const toolLogs = rig.readToolLogs();
// Verify edit tools WERE called
const editCalls = toolLogs.filter(
(log) =>
EDIT_TOOL_NAMES.has(log.toolRequest.name) && log.toolRequest.success,
);
expect(editCalls.length).toBeGreaterThanOrEqual(1);
// Verify file changed
const content = rig.readFile('app.ts');
expect(content).toContain('a + b');
},
});
/**
* Ensures that when the user asks "any bugs?" the agent does NOT
* automatically modify the file, but instead asks for permission.
*/
evalTest('USUALLY_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'should not edit when asking "any bugs"',
prompt: 'Any bugs in app.ts?',
files: FILES,
assert: async (rig) => {
const toolLogs = rig.readToolLogs();
// Verify NO edit tools called
const editCalls = toolLogs.filter((log) =>
EDIT_TOOL_NAMES.has(log.toolRequest.name),
);
expect(editCalls.length).toBe(0);
// Verify file unchanged
const content = rig.readFile('app.ts');
expect(content).toContain('a - b');
},
});
/**
* Ensures that when the user asks a general question, the agent does NOT
* automatically modify the file.
*/
evalTest('ALWAYS_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'should not edit files when asked a general question',
prompt: 'How does app.ts work?',
files: FILES,
assert: async (rig) => {
const toolLogs = rig.readToolLogs();
// Verify NO edit tools called
const editCalls = toolLogs.filter((log) =>
EDIT_TOOL_NAMES.has(log.toolRequest.name),
);
expect(editCalls.length).toBe(0);
// Verify file unchanged
const content = rig.readFile('app.ts');
expect(content).toContain('a - b');
},
});
/**
* Ensures that when the user asks a question about style, the agent does NOT
* automatically modify the file.
*/
evalTest('ALWAYS_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'should not edit files when asked about style',
prompt: 'Is app.ts following good style?',
files: FILES,
assert: async (rig, result) => {
const toolLogs = rig.readToolLogs();
// Verify NO edit tools called
const editCalls = toolLogs.filter((log) =>
EDIT_TOOL_NAMES.has(log.toolRequest.name),
);
expect(editCalls.length).toBe(0);
// Verify file unchanged
const content = rig.readFile('app.ts');
expect(content).toContain('a - b');
},
});
/**
* Ensures that when the user points out an issue but doesn't ask for a fix,
* the agent does NOT automatically modify the file.
*/
evalTest('USUALLY_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'should not edit files when user notes an issue',
prompt: 'The add function subtracts numbers.',
files: FILES,
params: { timeout: 20000 }, // 20s timeout
assert: async (rig) => {
const toolLogs = rig.readToolLogs();
// Verify NO edit tools called
const editCalls = toolLogs.filter((log) =>
EDIT_TOOL_NAMES.has(log.toolRequest.name),
);
expect(editCalls.length).toBe(0);
// Verify file unchanged
const content = rig.readFile('app.ts');
expect(content).toContain('a - b');
},
});
});
|