Download lib/llm/__tests__/agent-loop.test.ts from Leon4gr45/builder: direct link, hf CLI and curl.
- Browser
- Download file 37.3 kB
-
https://huggingface.co/spaces/Leon4gr45/builder/resolve/main/lib/llm/__tests__/agent-loop.test.ts
- Command line
-
hf download hf://spaces/Leon4gr45/builder/lib/llm/__tests__/agent-loop.test.ts
-
curl -L -o agent-loop.test.ts https://huggingface.co/spaces/Leon4gr45/builder/resolve/main/lib/llm/__tests__/agent-loop.test.ts
37.3 kB
| import { describe, it, expect, vi } from 'vitest'; | |
| import { AgentLoop } from '../core/agent-loop'; | |
| import type { | |
| ProviderAdapter, | |
| ToolExecutor, | |
| ContextManager, | |
| ProgressReporter, | |
| CostTracker, | |
| AgentLoopConfig, | |
| ParsedResponse, | |
| ToolResult, | |
| ToolCall, | |
| ToolDef, | |
| Message, | |
| UsageInfo, | |
| } from '../core/types'; | |
| // --- Mock factories --- | |
| function createMockProvider(responses: ParsedResponse[]): ProviderAdapter { | |
| let callIndex = 0; | |
| return { | |
| call: vi.fn(async () => { | |
| if (callIndex >= responses.length) { | |
| return { content: '' }; | |
| } | |
| return responses[callIndex++]; | |
| }), | |
| getModel: () => 'test-model', | |
| getProvider: () => 'test-provider', | |
| supportsTools: () => true, | |
| }; | |
| } | |
| function createMockExecutor(results?: Map<string, ToolResult>): ToolExecutor { | |
| return { | |
| execute: vi.fn(async (toolCall: ToolCall) => { | |
| if (results && results.has(toolCall.id)) { | |
| return results.get(toolCall.id)!; | |
| } | |
| return { | |
| tool_call_id: toolCall.id, | |
| content: 'OK', | |
| success: true, | |
| }; | |
| }), | |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }] as ToolDef[], | |
| }; | |
| } | |
| function createMockContext(): ContextManager { | |
| const messages: Message[] = []; | |
| return { | |
| getMessages: () => messages, | |
| getSanitizedMessages: () => messages, | |
| setSystemPrompt: vi.fn((prompt: string) => { | |
| if (messages.length > 0 && messages[0].role === 'system') { | |
| messages[0] = { role: 'system', content: prompt }; | |
| } else { | |
| messages.unshift({ role: 'system', content: prompt }); | |
| } | |
| }), | |
| addUserMessage: vi.fn((content: string) => { | |
| messages.push({ role: 'user', content }); | |
| }), | |
| addAssistantTurn: vi.fn((response: ParsedResponse) => { | |
| messages.push({ | |
| role: 'assistant', | |
| content: response.content || '', | |
| ...(response.toolCalls?.length ? { tool_calls: response.toolCalls } : {}), | |
| }); | |
| }), | |
| addToolResults: vi.fn((results: ToolResult[]) => { | |
| for (const r of results) { | |
| messages.push({ role: 'tool', content: r.content, tool_call_id: r.tool_call_id }); | |
| } | |
| }), | |
| importMessages: vi.fn(), | |
| needsCompaction: () => false, | |
| compact: vi.fn(async () => undefined), | |
| getTokenEstimate: () => 1000, | |
| getCompactionCount: () => 0, | |
| }; | |
| } | |
| function createMockProgress(): ProgressReporter { | |
| return { onEvent: vi.fn() }; | |
| } | |
| function createMockCost(): CostTracker { | |
| return { | |
| record: vi.fn(), | |
| getTurnCost: () => 0, | |
| getTotalCost: () => 0, | |
| getTotalUsage: () => ({ promptTokens: 0, completionTokens: 0, totalTokens: 0 }), | |
| resetTurn: vi.fn(), | |
| }; | |
| } | |
| function defaultConfig(overrides?: Partial<AgentLoopConfig>): AgentLoopConfig { | |
| return { | |
| maxIterations: 20, | |
| maxNudges: 2, | |
| maxDuplicateToolCalls: 3, | |
| agentType: 'orchestrator', | |
| isReadOnly: false, | |
| ...overrides, | |
| }; | |
| } | |
| // --- Tests --- | |
| describe('AgentLoop', () => { | |
| it('completes when model returns statusComplete signal', async () => { | |
| // 1. Provider returns a tool call, 2. Provider returns text (after status signal) | |
| const toolCall: ToolCall = { | |
| id: 'tc1', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"build site\\" --done \\"all\\" --remaining \\"none\\" --complete"}' }, | |
| }; | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [toolCall] }, | |
| { content: 'All done!' }, | |
| ]); | |
| const toolResult: ToolResult = { | |
| tool_call_id: 'tc1', | |
| content: 'Status recorded.', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 'build site', done: 'all', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }; | |
| const results = new Map([['tc1', toolResult]]); | |
| const executor = createMockExecutor(results); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig(); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor, | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('Build me a site'); | |
| expect(result.success).toBe(true); | |
| expect(result.toolCount).toBe(1); | |
| expect(result.turnCount).toBeGreaterThanOrEqual(1); | |
| }); | |
| it('stops after maxNudges when model returns text without tool calls', async () => { | |
| // Model always returns text, never tools — should nudge then stop | |
| const provider = createMockProvider([ | |
| { content: 'Let me think about that...' }, | |
| { content: 'Still thinking...' }, | |
| { content: 'Almost there...' }, | |
| { content: 'One more thought...' }, | |
| { content: 'Final thought.' }, | |
| ]); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig({ maxNudges: 2 }); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('Do something'); | |
| // First response = text, nudge 1, second response = text, nudge 2, third response = text, break | |
| expect(result.success).toBe(false); | |
| expect(result.summary).toContain('nudge'); | |
| }); | |
| it('nudges toward bash and retries when the model calls a non-bash tool (stream early-aborted)', async () => { | |
| const statusTool: ToolCall = { | |
| id: 'tc1', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"x\\" --done \\"y\\" --remaining \\"none\\" --complete"}' }, | |
| }; | |
| // 1) stream aborted on a 'cat' tool name, 2) retry uses bash correctly and completes | |
| const provider = createMockProvider([ | |
| { invalidToolName: 'cat' }, | |
| { content: '', toolCalls: [statusTool] }, | |
| { content: 'done' }, | |
| ]); | |
| const results = new Map([['tc1', { | |
| tool_call_id: 'tc1', | |
| content: 'Status recorded.', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 'x', done: 'y', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| } as ToolResult]]); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor: createMockExecutor(results), | |
| context, | |
| progress, | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('do it'); | |
| expect(progress.onEvent).toHaveBeenCalledWith('malformed_tool_call', expect.objectContaining({ invalidToolName: 'cat' })); | |
| const nudges = (context.addUserMessage as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0] as string); | |
| expect(nudges.some((m) => m.includes('cat') && m.includes('bash'))).toBe(true); | |
| expect(result.success).toBe(true); | |
| }); | |
| it('stops (does not loop forever) when the model keeps calling a non-bash tool', async () => { | |
| const provider = createMockProvider(Array.from({ length: 8 }, () => ({ invalidToolName: 'cat' } as ParsedResponse))); | |
| const progress = createMockProgress(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ maxNudges: 2 }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context: createMockContext(), | |
| progress, | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('do it'); | |
| expect(result.success).toBe(false); | |
| // The wrong-tool nudge fired (capped at MAX_MALFORMED_RETRIES) before falling through. | |
| const malformed = (progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.filter((c) => c[0] === 'malformed_tool_call'); | |
| expect(malformed.length).toBeGreaterThanOrEqual(2); | |
| }); | |
| it('interview agent pauses (awaiting_user) on a prose question instead of nudging', async () => { | |
| // A prose question is content with no tool call and no status — the | |
| // interview must hand control back to the user, not nudge toward completion. | |
| const provider = createMockProvider([ | |
| { content: "What's your company's name?" }, | |
| ]); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ agentType: 'interview' }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context: createMockContext(), | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('Understand a company'); | |
| expect(result.exitReason).toBe('awaiting_user'); | |
| expect(result.success).toBe(true); | |
| }); | |
| it('interview agent does not pause on a reasoning-only turn (no user-facing content)', async () => { | |
| // Reasoning without content is not a question — it should retry/nudge, not pause. | |
| const provider = createMockProvider([ | |
| { reasoningDetails: [{ type: 'reasoning.text', text: 'thinking' }] as ParsedResponse['reasoningDetails'] }, | |
| ]); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ agentType: 'interview', maxNudges: 1 }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context: createMockContext(), | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('Understand a company'); | |
| expect(result.exitReason).not.toBe('awaiting_user'); | |
| }); | |
| it('detects duplicate tool calls and stops after maxDuplicateToolCalls', async () => { | |
| const duplicateCall: ToolCall = { | |
| id: 'tc-dup', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"ls"}' }, | |
| }; | |
| // Same tool call repeated multiple times | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc1' }] }, | |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc2' }] }, | |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc3' }] }, | |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc4' }] }, | |
| ]); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig({ maxDuplicateToolCalls: 3 }); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('Do the thing'); | |
| expect(result.success).toBe(false); | |
| expect(result.summary).toContain('loop'); | |
| }); | |
| it('calls completionGate before accepting completion', async () => { | |
| let gateCallCount = 0; | |
| const completionGate = vi.fn(async () => { | |
| gateCallCount++; | |
| // First call returns error (blocks completion), second returns null (allows it) | |
| if (gateCallCount === 1) return 'Runtime error: undefined is not a function'; | |
| return null; | |
| }); | |
| // Use different command strings so they don't trigger duplicate detection | |
| const toolCall1: ToolCall = { | |
| id: 'tc1', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"build\\" --done \\"all\\" --remaining \\"none\\" --complete"}' }, | |
| }; | |
| const toolCall2: ToolCall = { | |
| id: 'tc2', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"build site\\" --done \\"everything\\" --remaining \\"none\\" --complete"}' }, | |
| }; | |
| const toolResult1: ToolResult = { | |
| tool_call_id: 'tc1', | |
| content: 'Status recorded.', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 'build', done: 'all', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }; | |
| const toolResult2: ToolResult = { | |
| tool_call_id: 'tc2', | |
| content: 'Status recorded.', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 'build site', done: 'everything', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }; | |
| // First iteration: tool call with statusComplete. Gate blocks. | |
| // Second iteration: different tool call with statusComplete. Gate allows. | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [toolCall1] }, | |
| { content: '', toolCalls: [toolCall2] }, | |
| ]); | |
| const results = new Map([ | |
| ['tc1', toolResult1], | |
| ['tc2', toolResult2], | |
| ]); | |
| const executor = createMockExecutor(results); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig({ completionGate }); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor, | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('Build it'); | |
| expect(completionGate).toHaveBeenCalledTimes(2); | |
| expect(result.success).toBe(true); | |
| }); | |
| it('handles provider error via onPausableError with continue', async () => { | |
| let callCount = 0; | |
| const provider: ProviderAdapter = { | |
| call: vi.fn(async () => { | |
| callCount++; | |
| if (callCount === 1) { | |
| throw new Error('Rate limited'); | |
| } | |
| // Second call succeeds with a tool call that signals complete | |
| return { | |
| content: '', | |
| toolCalls: [{ | |
| id: 'tc1', | |
| type: 'function' as const, | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"x\\" --done \\"x\\" --remaining \\"none\\" --complete"}' }, | |
| }], | |
| }; | |
| }), | |
| getModel: () => 'test-model', | |
| getProvider: () => 'test-provider', | |
| supportsTools: () => true, | |
| }; | |
| const onPausableError = vi.fn(async () => 'continue' as const); | |
| const toolResult: ToolResult = { | |
| tool_call_id: 'tc1', | |
| content: 'Status recorded.', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 'x', done: 'x', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }; | |
| const results = new Map([['tc1', toolResult]]); | |
| const executor = createMockExecutor(results); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig({ onPausableError }); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor, | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('Do it'); | |
| expect(onPausableError).toHaveBeenCalledTimes(1); | |
| expect(result.success).toBe(true); | |
| }); | |
| it('explore agent exits immediately without needing status', async () => { | |
| const provider = createMockProvider([ | |
| { content: 'Here is the analysis of the codebase...' }, | |
| ]); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig({ agentType: 'explore' }); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('Analyze this'); | |
| expect(result.success).toBe(true); | |
| expect(result.turnCount).toBe(1); | |
| // Should not have nudged — immediate exit | |
| const events = (progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); | |
| expect(events).not.toContain('nudge'); | |
| }); | |
| it('handles onPausableError returning stop', async () => { | |
| const provider: ProviderAdapter = { | |
| call: vi.fn(async () => { | |
| throw new Error('Fatal error'); | |
| }), | |
| getModel: () => 'test-model', | |
| getProvider: () => 'test-provider', | |
| supportsTools: () => true, | |
| }; | |
| const onPausableError = vi.fn(async () => 'stop' as const); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig({ onPausableError }); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('Do it'); | |
| expect(onPausableError).toHaveBeenCalledTimes(1); | |
| expect(result.success).toBe(false); | |
| expect(result.summary).toContain('error'); | |
| }); | |
| it('extracts tool calls from text for models without native tool support', async () => { | |
| const provider: ProviderAdapter = { | |
| call: vi.fn(async () => ({ | |
| content: '```bash\nls -la\n```', | |
| toolCalls: undefined, | |
| })), | |
| getModel: () => 'test-model', | |
| getProvider: () => 'test-provider', | |
| supportsTools: () => false, // no native tool support | |
| }; | |
| // After extraction, the tool call signals complete | |
| const executor: ToolExecutor = { | |
| execute: vi.fn(async (toolCall: ToolCall) => ({ | |
| tool_call_id: toolCall.id, | |
| content: 'files listed', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 'x', done: 'x', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| })), | |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], | |
| }; | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig(); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor, | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| const result = await loop.run('List files'); | |
| expect(executor.execute).toHaveBeenCalled(); | |
| expect(result.toolCount).toBe(1); | |
| }); | |
| it('stops when stop() is called', async () => { | |
| let callCount = 0; | |
| const provider: ProviderAdapter = { | |
| call: vi.fn(async () => { | |
| callCount++; | |
| return { | |
| content: '', | |
| toolCalls: [{ | |
| id: `tc${callCount}`, | |
| type: 'function' as const, | |
| function: { name: 'bash', arguments: `{"command":"echo ${callCount}"}` }, | |
| }], | |
| }; | |
| }), | |
| getModel: () => 'test-model', | |
| getProvider: () => 'test-provider', | |
| supportsTools: () => true, | |
| }; | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const config = defaultConfig(); | |
| const loop = new AgentLoop({ | |
| config, | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| // Stop after first iteration | |
| (provider.call as ReturnType<typeof vi.fn>).mockImplementation(async () => { | |
| if (callCount > 0) loop.stop(); | |
| callCount++; | |
| return { | |
| content: '', | |
| toolCalls: [{ | |
| id: `tc${callCount}`, | |
| type: 'function' as const, | |
| function: { name: 'bash', arguments: `{"command":"echo ${callCount}"}` }, | |
| }], | |
| }; | |
| }); | |
| const result = await loop.run('Do something'); | |
| expect(result.success).toBe(false); | |
| expect(result.summary).toContain('Stopped'); | |
| }); | |
| it('triggers compaction using token estimate when provider reports no usage', async () => { | |
| const toolCall: ToolCall = { | |
| id: 'tc1', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, | |
| }; | |
| // Provider returns no usage (simulating providers that don't support stream_options) | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [toolCall] }, | |
| { content: 'Done.' }, | |
| ]); | |
| const context = createMockContext(); | |
| // getTokenEstimate returns a value above threshold — should trigger compaction | |
| context.getTokenEstimate = () => 999999; | |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; | |
| const toolResult: ToolResult = { | |
| tool_call_id: 'tc1', | |
| content: 'Status recorded.', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }; | |
| const executor = createMockExecutor(new Map([['tc1', toolResult]])); | |
| const progress = createMockProgress(); | |
| // promptTokens=0 simulates missing usage | |
| const cost = createMockCost(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor, | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| await loop.run('Build something'); | |
| expect(context.compact).toHaveBeenCalled(); | |
| }); | |
| it('uses reported promptTokens for compaction check when available', async () => { | |
| const toolCall: ToolCall = { | |
| id: 'tc1', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, | |
| }; | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [toolCall], usage: { promptTokens: 90000, completionTokens: 500, totalTokens: 90500 } }, | |
| { content: 'Done.' }, | |
| ]); | |
| const context = createMockContext(); | |
| context.getTokenEstimate = () => 5000; // Low estimate — should NOT be used | |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; | |
| const toolResult: ToolResult = { | |
| tool_call_id: 'tc1', | |
| content: 'ok', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }; | |
| const executor = createMockExecutor(new Map([['tc1', toolResult]])); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| cost.getTotalUsage = () => ({ promptTokens: 90000, completionTokens: 500, totalTokens: 90500 }); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor, | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| await loop.run('Build something'); | |
| expect(context.compact).toHaveBeenCalled(); | |
| }); | |
| it('does not compact when both promptTokens and estimate are below threshold', async () => { | |
| const toolCall: ToolCall = { | |
| id: 'tc1', | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, | |
| }; | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [toolCall] }, | |
| { content: 'Done.' }, | |
| ]); | |
| const context = createMockContext(); | |
| context.getTokenEstimate = () => 5000; | |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; | |
| const toolResult: ToolResult = { | |
| tool_call_id: 'tc1', | |
| content: 'ok', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }; | |
| const executor = createMockExecutor(new Map([['tc1', toolResult]])); | |
| const progress = createMockProgress(); | |
| const cost = createMockCost(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor, | |
| context, | |
| progress, | |
| cost, | |
| }); | |
| await loop.run('Build something'); | |
| expect(context.compact).not.toHaveBeenCalled(); | |
| }); | |
| }); | |
| describe('AgentLoop history integrity and retry safeguards', () => { | |
| const statusToolCall = (id: string): ToolCall => ({ | |
| id, | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, | |
| }); | |
| const statusResultFor = (id: string): ToolResult => ({ | |
| tool_call_id: id, | |
| content: 'ok', | |
| success: true, | |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, | |
| }); | |
| it('sends sanitized messages (orphan repair) to the provider', async () => { | |
| const sanitized: Message[] = [{ role: 'user', content: 'SANITIZED' }]; | |
| const context = createMockContext(); | |
| context.getSanitizedMessages = () => sanitized; | |
| const provider = createMockProvider([{ content: 'findings' }]); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ agentType: 'explore' }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| await loop.run('hi'); | |
| const sentMessages = (provider.call as ReturnType<typeof vi.fn>).mock.calls[0][0].messages; | |
| expect(sentMessages).toBe(sanitized); | |
| }); | |
| it('uses its own last response promptTokens for compaction, not the shared cost tracker', async () => { | |
| // Shared tracker reports a small (child-contaminated) value; this loop's own | |
| // response reported 90000 prompt tokens — compaction must use 90000. | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [statusToolCall('tc1')], usage: { promptTokens: 90000, completionTokens: 500, totalTokens: 90500 } }, | |
| { content: 'Done.' }, | |
| ]); | |
| const context = createMockContext(); | |
| context.getTokenEstimate = () => 5000; | |
| const needsCompaction = vi.fn((tokenCount: number) => tokenCount >= 80000); | |
| context.needsCompaction = needsCompaction; | |
| const cost = createMockCost(); | |
| cost.getTotalUsage = () => ({ promptTokens: 50, completionTokens: 0, totalTokens: 50 }); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), | |
| context, | |
| progress: createMockProgress(), | |
| cost, | |
| }); | |
| await loop.run('Build something'); | |
| expect(needsCompaction).toHaveBeenCalledWith(90000); | |
| expect(context.compact).toHaveBeenCalled(); | |
| }); | |
| it('commits executed tool results to context before terminating on loop detection', async () => { | |
| const dup = (id: string): ToolCall => ({ | |
| id, | |
| type: 'function', | |
| function: { name: 'bash', arguments: '{"command":"ls"}' }, | |
| }); | |
| // One batch: first call executes, the next 3 identical calls trip the | |
| // duplicate cap (maxDuplicateToolCalls: 3) mid-batch. | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [dup('a'), dup('b'), dup('c'), dup('d')] }, | |
| ]); | |
| const context = createMockContext(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ maxDuplicateToolCalls: 3 }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('Do something'); | |
| expect(result.summary).toContain('loop'); | |
| // The assistant turn and the executed result for 'a' must be in history | |
| expect(context.addAssistantTurn).toHaveBeenCalledTimes(1); | |
| expect(context.addToolResults).toHaveBeenCalled(); | |
| const committed = (context.addToolResults as ReturnType<typeof vi.fn>).mock.calls[0][0] as ToolResult[]; | |
| expect(committed.some(r => r.tool_call_id === 'a' && r.success)).toBe(true); | |
| }); | |
| it('passes a per-turn turnId to the tool executor context', async () => { | |
| const call = (id: string, cmd: string): ToolCall => ({ | |
| id, | |
| type: 'function', | |
| function: { name: 'bash', arguments: `{"command":"${cmd}"}` }, | |
| }); | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [call('t1', 'ls')] }, | |
| { content: '', toolCalls: [call('t2', 'status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete')] }, | |
| { content: 'Done.' }, | |
| ]); | |
| const turnIds: Array<number | undefined> = []; | |
| const executor: ToolExecutor = { | |
| execute: vi.fn(async (toolCall: ToolCall, ctx) => { | |
| turnIds.push(ctx.turnId); | |
| if (toolCall.id === 't2') return statusResultFor('t2'); | |
| return { tool_call_id: toolCall.id, content: 'OK', success: true }; | |
| }), | |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], | |
| }; | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor, | |
| context: createMockContext(), | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| await loop.run('Do something'); | |
| expect(turnIds).toEqual([1, 2]); | |
| }); | |
| it('passes its abort signal to compact()', async () => { | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [statusToolCall('tc1')], usage: { promptTokens: 90000, completionTokens: 1, totalTokens: 90001 } }, | |
| { content: 'Done.' }, | |
| ]); | |
| const context = createMockContext(); | |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), | |
| context, | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| await loop.run('Build something'); | |
| const compactOpts = (context.compact as ReturnType<typeof vi.fn>).mock.calls[0][1]; | |
| expect(compactOpts?.signal).toBeInstanceOf(AbortSignal); | |
| }); | |
| it('persists reasoning-only turns and injects a targeted retry instead of the status nudge', async () => { | |
| const reasoningOnly = { | |
| content: '', | |
| reasoningDetails: [{ type: 'thinking', text: 'I should check index.html first.' }], | |
| }; | |
| const provider = createMockProvider([ | |
| reasoningOnly, | |
| { content: '', toolCalls: [statusToolCall('tc1')] }, | |
| { content: 'Done.' }, | |
| ]); | |
| const context = createMockContext(); | |
| const progress = createMockProgress(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), | |
| context, | |
| progress, | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('Build a landing page'); | |
| // The reasoning-only assistant turn must be persisted with its reasoning details | |
| const assistantTurns = (context.addAssistantTurn as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); | |
| expect(assistantTurns.some(r => r.reasoningDetails?.length && !r.content)).toBe(true); | |
| // The injected user message must be the targeted retry, not the status nudge | |
| const userMessages = (context.addUserMessage as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); | |
| const retryMsg = userMessages.find((m: string) => typeof m === 'string' && m.includes('only reasoning')); | |
| expect(retryMsg).toBeDefined(); | |
| expect(userMessages.some((m: string) => typeof m === 'string' && m.includes('status command'))).toBe(false); | |
| expect((progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.some(c => c[0] === 'reasoning_only_retry')).toBe(true); | |
| expect(result.success).toBe(true); | |
| }); | |
| it('falls back to status nudges after reasoning-only retries are exhausted', async () => { | |
| const reasoningOnly = () => ({ | |
| content: '', | |
| reasoningDetails: [{ type: 'thinking', text: 'Thinking again from scratch.' }], | |
| }); | |
| const provider = createMockProvider([ | |
| reasoningOnly(), reasoningOnly(), reasoningOnly(), reasoningOnly(), reasoningOnly(), | |
| ]); | |
| const progress = createMockProgress(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ maxNudges: 1 }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context: createMockContext(), | |
| progress, | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('Build a landing page'); | |
| const retryEvents = (progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.filter(c => c[0] === 'reasoning_only_retry'); | |
| expect(retryEvents).toHaveLength(2); | |
| expect(result.success).toBe(false); | |
| expect(result.exitReason).toBe('nudge_exhaustion'); | |
| }); | |
| it('keeps reasoning details on assistant turns that have content', async () => { | |
| const provider = createMockProvider([ | |
| { content: 'Here is my answer.', reasoningDetails: [{ type: 'thinking', text: 'Considered options.' }] }, | |
| { content: '', toolCalls: [statusToolCall('tc1')] }, | |
| { content: 'Done.' }, | |
| ]); | |
| const context = createMockContext(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), | |
| context, | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| await loop.run('Question'); | |
| const assistantTurns = (context.addAssistantTurn as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); | |
| const withContent = assistantTurns.find(r => r.content === 'Here is my answer.'); | |
| expect(withContent?.reasoningDetails?.length).toBe(1); | |
| }); | |
| it('wraps synthetic harness messages in automated_reminder tags, but never the real user prompt', async () => { | |
| // Run produces: reasoning-only retry, then text-only nudges | |
| const provider = createMockProvider([ | |
| { content: '', reasoningDetails: [{ type: 'thinking', text: 'hmm' }] }, | |
| { content: 'text without status' }, | |
| { content: 'more text' }, | |
| ]); | |
| const context = createMockContext(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ maxNudges: 1 }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context, | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| await loop.run('Build a landing page'); | |
| const userMessages = (context.addUserMessage as ReturnType<typeof vi.fn>).mock.calls | |
| .map(c => c[0]) | |
| .filter((m): m is string => typeof m === 'string'); | |
| // The real user prompt is untouched | |
| expect(userMessages[0]).toBe('Build a landing page'); | |
| // Every synthetic injection is wrapped | |
| const synthetic = userMessages.slice(1); | |
| expect(synthetic.length).toBeGreaterThan(0); | |
| for (const msg of synthetic) { | |
| expect(msg.startsWith('<automated_reminder>')).toBe(true); | |
| expect(msg.endsWith('</automated_reminder>')).toBe(true); | |
| } | |
| }); | |
| it('replaces large repeated tool outputs with a dedup marker', async () => { | |
| const bigContent = 'SKILL FILE CONTENT '.repeat(50); // ~950 chars | |
| const call = (id: string, cmd: string): ToolCall => ({ | |
| id, | |
| type: 'function', | |
| function: { name: 'bash', arguments: JSON.stringify({ command: cmd }) }, | |
| }); | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [call('t1', 'cat /skill.md')] }, | |
| { content: '', toolCalls: [call('t2', 'head -n 999 /skill.md')] }, | |
| { content: '', toolCalls: [call('t3', 'status --task "t" --done "d" --remaining "none" --complete')] }, | |
| { content: 'Done.' }, | |
| ]); | |
| const executor: ToolExecutor = { | |
| execute: vi.fn(async (toolCall: ToolCall) => { | |
| if (toolCall.id === 't3') return statusResultFor('t3'); | |
| return { tool_call_id: toolCall.id, content: bigContent, success: true }; | |
| }), | |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], | |
| }; | |
| const context = createMockContext(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor, | |
| context, | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| await loop.run('Do something'); | |
| const toolBatches = (context.addToolResults as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0] as ToolResult[]); | |
| // First read keeps full content | |
| expect(toolBatches[0][0].content).toBe(bigContent); | |
| // Identical repeat is replaced with a marker | |
| expect(toolBatches[1][0].content).not.toBe(bigContent); | |
| expect(toolBatches[1][0].content).toContain('identical'); | |
| expect(toolBatches[1][0].success).toBe(true); | |
| }); | |
| it('does not dedup small repeated tool outputs', async () => { | |
| const call = (id: string, cmd: string): ToolCall => ({ | |
| id, | |
| type: 'function', | |
| function: { name: 'bash', arguments: JSON.stringify({ command: cmd }) }, | |
| }); | |
| const provider = createMockProvider([ | |
| { content: '', toolCalls: [call('t1', 'mkdir /a')] }, | |
| { content: '', toolCalls: [call('t2', 'mkdir /b')] }, | |
| { content: '', toolCalls: [call('t3', 'status --task "t" --done "d" --remaining "none" --complete')] }, | |
| { content: 'Done.' }, | |
| ]); | |
| const executor: ToolExecutor = { | |
| execute: vi.fn(async (toolCall: ToolCall) => { | |
| if (toolCall.id === 't3') return statusResultFor('t3'); | |
| return { tool_call_id: toolCall.id, content: 'Command succeeded with no output', success: true }; | |
| }), | |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], | |
| }; | |
| const context = createMockContext(); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig(), | |
| provider, | |
| executor, | |
| context, | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| await loop.run('Do something'); | |
| const toolBatches = (context.addToolResults as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0] as ToolResult[]); | |
| expect(toolBatches[1][0].content).toBe('Command succeeded with no output'); | |
| }); | |
| it('reports the exit reason in the loop result', async () => { | |
| const provider = createMockProvider([ | |
| { content: 'text only' }, | |
| { content: 'more text' }, | |
| { content: 'final text' }, | |
| ]); | |
| const loop = new AgentLoop({ | |
| config: defaultConfig({ maxNudges: 2 }), | |
| provider, | |
| executor: createMockExecutor(), | |
| context: createMockContext(), | |
| progress: createMockProgress(), | |
| cost: createMockCost(), | |
| }); | |
| const result = await loop.run('Do something'); | |
| expect(result.exitReason).toBe('nudge_exhaustion'); | |
| }); | |
| }); | |