File size: 4,467 Bytes
c6d079d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | /**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import {
type EvalPolicy,
runEval,
prepareLogDir,
withEvalRetries,
prepareWorkspace,
type BaseEvalCase,
} from './test-helper.js';
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { randomUUID } from 'node:crypto';
import { vi } from 'vitest';
import {
Config,
type ConfigParameters,
AuthType,
ApprovalMode,
createPolicyEngineConfig,
ExtensionLoader,
IntegrityDataStatus,
makeFakeConfig,
type GeminiCLIExtension,
} from '@google/gemini-cli-core';
import { createMockSettings } from '../packages/cli/src/test-utils/settings.js';
// A minimal mock ExtensionManager to bypass integrity checks
class MockExtensionManager extends ExtensionLoader {
override getExtensions(): GeminiCLIExtension[] {
return [];
}
setRequestConsent = (): void => {};
setRequestSetting = (): void => {};
integrityManager = {
verifyExtensionIntegrity: async (): Promise<IntegrityDataStatus> =>
IntegrityDataStatus.VERIFIED,
storeExtensionIntegrity: async (): Promise<void> => undefined,
};
}
export interface ComponentEvalCase extends BaseEvalCase {
configOverrides?: Partial<ConfigParameters>;
setup?: (config: Config) => Promise<void>;
assert: (config: Config) => Promise<void>;
}
export class ComponentRig {
public config: Config | undefined;
public testDir: string;
public homeDir: string;
public sessionId: string;
constructor(
private options: { configOverrides?: Partial<ConfigParameters> } = {},
) {
const uniqueId = randomUUID();
this.testDir = fs.mkdtempSync(
path.join(os.tmpdir(), `gemini-component-rig-${uniqueId.slice(0, 8)}-`),
);
this.homeDir = fs.mkdtempSync(
path.join(os.tmpdir(), `gemini-component-home-${uniqueId.slice(0, 8)}-`),
);
this.sessionId = `test-session-${uniqueId}`;
}
async initialize() {
const settings = createMockSettings();
const policyEngineConfig = await createPolicyEngineConfig(
settings.merged,
ApprovalMode.DEFAULT,
);
const configParams: ConfigParameters = {
sessionId: this.sessionId,
targetDir: this.testDir,
cwd: this.testDir,
debugMode: false,
model: 'test-model',
interactive: false,
approvalMode: ApprovalMode.DEFAULT,
policyEngineConfig,
enableEventDrivenScheduler: false, // Don't need scheduler for direct component tests
extensionLoader: new MockExtensionManager(),
useAlternateBuffer: false,
...this.options.configOverrides,
};
this.config = makeFakeConfig(configParams);
await this.config.initialize();
// Refresh auth using USE_GEMINI to initialize the real BaseLlmClient.
// This must happen BEFORE stubbing GEMINI_CLI_HOME because OAuth credential
// lookup resolves through homedir() → GEMINI_CLI_HOME.
await this.config.refreshAuth(AuthType.USE_GEMINI);
// Isolate storage paths (session files, skills, extraction state) by
// pointing GEMINI_CLI_HOME at a per-test temp directory. Storage resolves
// global paths through `homedir()` which reads this env var. This is set
// after auth so credential lookup uses the real home directory.
vi.stubEnv('GEMINI_CLI_HOME', this.homeDir);
}
async cleanup() {
await this.config?.dispose();
vi.unstubAllEnvs();
fs.rmSync(this.testDir, { recursive: true, force: true });
fs.rmSync(this.homeDir, { recursive: true, force: true });
}
}
/**
* A helper for running behavioral evaluations directly against backend components.
* It provides a fully initialized Config with real API access, bypassing the UI.
*/
export function componentEvalTest(
policy: EvalPolicy,
evalCase: ComponentEvalCase,
) {
const fn = async () => {
await withEvalRetries(evalCase.name, async () => {
const rig = new ComponentRig({
configOverrides: evalCase.configOverrides,
});
await prepareLogDir(evalCase.name);
try {
await rig.initialize();
if (evalCase.files) {
await prepareWorkspace(rig.testDir, rig.testDir, evalCase.files);
}
if (evalCase.setup) {
await evalCase.setup(rig.config!);
}
await evalCase.assert(rig.config!);
} finally {
await rig.cleanup();
}
});
};
runEval(policy, evalCase, fn, (evalCase.timeout ?? 60000) + 10000);
}
|