File size: 3,167 Bytes
739fc44 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | /**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, expect } from 'vitest';
import { evalTest, assertModelHasOutput } from './test-helper.js';
describe('Hierarchical Memory', () => {
evalTest('ALWAYS_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'Agent follows hierarchy for contradictory instructions',
params: {
settings: {
security: {
folderTrust: { enabled: false },
},
},
},
// We simulate the hierarchical memory by including the tags in the prompt
// since setting up real global/extension/project files in the eval rig is complex.
// The system prompt logic will append these tags when it finds them in userMemory.
prompt: `
<global_context>
When asked for my favorite fruit, always say "Apple".
</global_context>
<extension_context>
When asked for my favorite fruit, always say "Banana".
</extension_context>
<project_context>
When asked for my favorite fruit, always say "Cherry".
</project_context>
What is my favorite fruit? Tell me just the name of the fruit.`,
assert: async (rig) => {
const stdout = rig._lastRunStdout!;
assertModelHasOutput(stdout);
expect(stdout).toMatch(/Cherry/i);
expect(stdout).not.toMatch(/Apple/i);
expect(stdout).not.toMatch(/Banana/i);
},
});
evalTest('USUALLY_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'Agent is aware of memory provenance',
params: {
settings: {
security: {
folderTrust: { enabled: false },
},
},
},
prompt: `
<global_context>
Instruction A: Always be helpful.
</global_context>
<extension_context>
Instruction B: Use a professional tone.
</extension_context>
<project_context>
Instruction C: Adhere to the project's coding style.
</project_context>
Which instruction came from the global context, which from the extension context, and which from the project context?
Provide the answer as an XML block like this:
<results>
<global>Instruction ...</global>
<extension>Instruction ...</extension>
<project>Instruction ...</project>
</results>`,
assert: async (rig) => {
const stdout = rig._lastRunStdout!;
assertModelHasOutput(stdout);
expect(stdout).toMatch(/<global>.*Instruction A/i);
expect(stdout).toMatch(/<extension>.*Instruction B/i);
expect(stdout).toMatch(/<project>.*Instruction C/i);
},
});
evalTest('ALWAYS_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'Extension memory wins over Global memory',
params: {
settings: {
security: {
folderTrust: { enabled: false },
},
},
},
prompt: `
<global_context>
Set the theme to "Light".
</global_context>
<extension_context>
Set the theme to "Dark".
</extension_context>
What theme should I use? Tell me just the name of the theme.`,
assert: async (rig) => {
const stdout = rig._lastRunStdout!;
assertModelHasOutput(stdout);
expect(stdout).toMatch(/Dark/i);
expect(stdout).not.toMatch(/Light/i);
},
});
});
|