File size: 12,721 Bytes
6cd657d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 | """
ExecutionEngine Compression Tests
1. Context exceeding 200k β actual summary generated β tokens come down
2. Orphan tool outputs β don't crash the agent run
Run with: pytest tests/core/test_execution_engine_compression.py -v
"""
import sys
import os
import uuid
import json
import pytest
pytestmark = pytest.mark.filterwarnings("ignore::UserWarning")
from typing import List, Dict, Any
from unittest.mock import AsyncMock, MagicMock, patch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
# Set dummy env vars so module-level initializers don't crash in CI.
# These are never used β all external calls (sandbox, DB, LLM) are mocked.
os.environ.setdefault("DAYTONA_API_KEY", "test-key")
os.environ.setdefault("DAYTONA_SERVER_URL", "http://localhost")
os.environ.setdefault("DAYTONA_TARGET", "local")
os.environ.setdefault("MCP_CREDENTIAL_ENCRYPTION_KEY", "KEp9Zg9R1XO8EOcHoUH58dEkIQVJHIFKzKWKlpuQ6tY=")
os.environ.setdefault("SUPABASE_URL", "http://localhost:54321")
os.environ.setdefault("SUPABASE_ANON_KEY", "test-anon-key")
os.environ.setdefault("SUPABASE_SERVICE_ROLE_KEY", "test-service-key")
os.environ.setdefault("SUPABASE_JWT_SECRET", "test-jwt-secret")
from core.agents.pipeline.stateless.coordinator.execution import ExecutionEngine
try:
from core.agents.pipeline.stateless.context.archiver import ContextArchiver
HAS_ARCHIVER = True
except ImportError:
HAS_ARCHIVER = False
# --- Helpers ---
def make_user(content, **kw):
msg = {"role": "user", "content": content}
msg.update(kw)
return msg
def make_assistant(content, tool_calls=None, **kw):
msg = {"role": "assistant", "content": content}
if tool_calls:
msg["tool_calls"] = tool_calls
msg.update(kw)
return msg
def make_tool(content, tool_call_id=None, **kw):
msg = {"role": "tool", "content": content, "tool_call_id": tool_call_id or f"call_{uuid.uuid4().hex[:12]}"}
msg.update(kw)
return msg
def make_tool_call(name="some_tool"):
return {"id": f"call_{uuid.uuid4().hex[:12]}", "type": "function", "function": {"name": name, "arguments": "{}"}}
def big_content(n):
base = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
return (base * ((n // len(base)) + 1))[:n]
def _mock_sandbox():
"""Create a fake sandbox that stores files in memory."""
files = {}
sandbox = MagicMock()
async def upload_file(content, path):
if isinstance(content, bytes):
files[path] = content
else:
files[path] = content.encode("utf-8") if isinstance(content, str) else content
async def download_file(path):
if path in files:
return files[path]
raise FileNotFoundError(path)
async def create_folder(path, mode="755"):
pass # no-op
sandbox.fs.upload_file = upload_file
sandbox.fs.download_file = download_file
sandbox.fs.create_folder = create_folder
return sandbox, files
def _mock_llm_summary_response():
"""Return a fake ModelResponse with a valid summary JSON."""
import litellm
summary_json = json.dumps({
"summary": "User requested building a website. Agent searched for best practices and processed large pasted content including documentation and design specs.",
"topics": ["web development", "website building"],
"key_decisions": [{"decision": "Use React for frontend", "rationale": "User preference"}],
"facts": {
"project_name": "user-website",
"tech_stack": ["React", "Node.js"],
"current_goal": "Build a complete website"
}
})
return litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "test"}],
mock_response=summary_json,
)
# ============================================================================
# 1. Context > 200k β actual summary generated β tokens come down
# ============================================================================
@pytest.mark.skipif(not HAS_ARCHIVER, reason="ContextArchiver not available (main branch)")
class TestContextExceeds200k:
"""When context hits 200k tokens, compression should:
1. Call the archiver which calls the LLM to generate a real summary
2. Write archive files to sandbox
3. Compress working memory (truncate giant messages)
4. Return compressed messages with summary
Only the LLM call and sandbox filesystem are mocked β everything else
runs for real.
"""
@pytest.mark.asyncio
async def test_over_200k_generates_summary_and_compresses(self):
"""Full pipeline with actual summary generation (mocked LLM response)."""
engine = ExecutionEngine(MagicMock(), MagicMock())
engine._state.model_name = "test-model"
engine._state.thread_id = "test-thread"
engine._state.project_id = "test-project"
engine._state.account_id = "test-account"
engine._state.stream_key = "test-stream"
engine.TEST_THRESHOLD_OVERRIDE = 100 # Force trigger
# 10 messages with giant user pastes (the real scenario)
tc = make_tool_call("search")
msgs = [
make_user("Build me a website"),
make_assistant("Sure, let me search for best practices.", [tc]),
make_tool(big_content(50_000), tc["id"]),
make_assistant("Here's what I found."),
make_user(big_content(403_000)), # Giant paste β 403K chars
make_assistant("Processing that..."),
make_user(big_content(315_000)), # Another giant paste β 315K chars
make_assistant("Got it."),
make_user("Now finish it up"),
make_assistant("Done!"),
]
system = {"role": "system", "content": "You are helpful."}
# Mock sandbox filesystem (in-memory)
sandbox, files = _mock_sandbox()
mock_sandbox_info = MagicMock()
mock_sandbox_info.sandbox = sandbox
# Mock LLM to return a proper summary JSON
mock_llm_response = _mock_llm_summary_response()
async def mock_make_llm_call(**kwargs):
return mock_llm_response
async def mock_get_db_client():
return MagicMock()
mock_db_conn = MagicMock()
mock_db_conn.client = mock_get_db_client()
with patch("core.ai_models.model_manager.get_context_window", return_value=200_000), \
patch("core.services.supabase.DBConnection", return_value=mock_db_conn), \
patch("core.sandbox.resolver.resolve_sandbox", new_callable=AsyncMock, return_value=mock_sandbox_info), \
patch("core.services.llm.make_llm_api_call", side_effect=mock_make_llm_call), \
patch("core.cache.runtime_cache.set_cached_message_history", new_callable=AsyncMock), \
patch("core.agents.pipeline.ux_streaming.stream_summarizing", new_callable=AsyncMock):
result_msgs, _, did_compress = await engine._check_and_compress_if_needed(
msgs, tokens=200_000, system_prompt=system
)
# --- Compression triggered ---
assert did_compress is True
assert len(result_msgs) < len(msgs), "Should have fewer messages after compression"
# --- Summary is first message and contains actual LLM-generated content ---
summary_msg = result_msgs[0]
assert summary_msg.get("_is_summary_inline") is True
assert "website" in summary_msg["content"].lower(), \
"Summary should contain LLM-generated content about the website task"
# --- Giant user messages in working memory are truncated ---
for m in result_msgs:
if m["role"] == "user" and not m.get("_is_summary_inline"):
assert len(m["content"]) <= 5000, \
f"User message still {len(m['content'])} chars β should be truncated to ~4K"
# --- Giant tool outputs are truncated ---
for m in result_msgs:
if m["role"] == "tool":
assert len(m["content"]) <= 4000, \
f"Tool output still {len(m['content'])} chars β should be truncated"
# --- Archive files were written to sandbox ---
assert any("batch_001.md" in path for path in files), \
f"Summary batch file not written. Files: {list(files.keys())}"
assert any("manifest.json" in path for path in files), \
f"Manifest not written. Files: {list(files.keys())}"
# --- Summary file contains the LLM-generated summary ---
summary_file = [v for k, v in files.items() if "batch_001.md" in k][0]
summary_content = summary_file.decode("utf-8") if isinstance(summary_file, bytes) else summary_file
assert "website" in summary_content.lower(), \
"Archive summary file should contain LLM-generated summary about the website"
# ============================================================================
# 2. Orphan tool outputs β don't crash the agent run
# ============================================================================
class TestOrphanToolDontCrashAgent:
"""When messages have orphan tool results (no matching assistant tool_call),
execute_step should repair them and proceed without crashing.
This happens when context is reloaded from DB after compression β the
assistant that called the tool was archived, but the tool result wasn't.
"""
@pytest.mark.asyncio
async def test_orphan_tool_handled_in_execute_step(self):
"""execute_step detects orphan, repairs it, calls LLM successfully."""
from core.agents.pipeline.stateless.context.manager import ContextManager
state = MagicMock()
state.model_name = "test-model"
state.thread_id = "test-thread"
state.stream_key = "test-stream"
state.tool_schemas = [{"type": "function", "function": {"name": "test", "parameters": {}}}]
state.system_prompt = {"role": "system", "content": "You are helpful."}
# Orphan tool result β its matching assistant was archived
orphan_messages = [
make_tool("result from archived assistant call", "call_archived_001"),
make_user("continue working on X"),
make_assistant("Sure, working on X."),
]
state.get_messages.return_value = orphan_messages
state._messages = list(orphan_messages)
mock_layers = MagicMock()
mock_layers.to_messages.return_value = orphan_messages
mock_layers.total_messages = len(orphan_messages)
async def mock_process_response(response):
yield {"type": "text", "content": "done"}
processor = MagicMock()
processor.process_response = mock_process_response
engine = ExecutionEngine(state, processor)
async def mock_llm_stream():
yield {"choices": [{"delta": {"content": "hello"}}]}
mock_executor = MagicMock()
mock_executor.execute = AsyncMock(return_value=mock_llm_stream())
with patch.object(ContextManager, "extract_layers", return_value=mock_layers), \
patch.object(engine, "fast_token_count", new_callable=AsyncMock, return_value=500), \
patch("core.agents.pipeline.ux_streaming.stream_context_usage", new_callable=AsyncMock), \
patch.object(engine, "_check_and_compress_if_needed", new_callable=AsyncMock, return_value=(orphan_messages, 500, False)), \
patch("core.agents.pipeline.stateless.coordinator.execution.add_cache_control", side_effect=lambda x: x), \
patch("core.agents.pipeline.stateless.coordinator.execution.LLMExecutor", return_value=mock_executor):
chunks = []
async for chunk in engine.execute_step():
chunks.append(chunk)
# Agent should NOT have crashed
error_chunks = [c for c in chunks if isinstance(c, dict) and c.get("type") == "error"]
assert len(error_chunks) == 0, f"Agent crashed with errors: {error_chunks}"
# LLM should have been called
mock_executor.execute.assert_called_once()
# No orphan tool results sent to LLM
sent_messages = mock_executor.execute.call_args.kwargs["prepared_messages"]
for msg in sent_messages:
if msg.get("role") == "tool":
tid = msg["tool_call_id"]
has_parent = any(
tid in [t["id"] for t in m.get("tool_calls", [])]
for m in sent_messages if m.get("role") == "assistant"
)
assert has_parent, f"Orphan tool {tid} sent to LLM β would cause API error"
|