File size: 12,721 Bytes
6cd657d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
"""
ExecutionEngine Compression Tests

1. Context exceeding 200k β†’ actual summary generated β†’ tokens come down
2. Orphan tool outputs β†’ don't crash the agent run

Run with: pytest tests/core/test_execution_engine_compression.py -v
"""

import sys
import os
import uuid
import json
import pytest

pytestmark = pytest.mark.filterwarnings("ignore::UserWarning")
from typing import List, Dict, Any
from unittest.mock import AsyncMock, MagicMock, patch

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))

# Set dummy env vars so module-level initializers don't crash in CI.
# These are never used β€” all external calls (sandbox, DB, LLM) are mocked.
os.environ.setdefault("DAYTONA_API_KEY", "test-key")
os.environ.setdefault("DAYTONA_SERVER_URL", "http://localhost")
os.environ.setdefault("DAYTONA_TARGET", "local")
os.environ.setdefault("MCP_CREDENTIAL_ENCRYPTION_KEY", "KEp9Zg9R1XO8EOcHoUH58dEkIQVJHIFKzKWKlpuQ6tY=")
os.environ.setdefault("SUPABASE_URL", "http://localhost:54321")
os.environ.setdefault("SUPABASE_ANON_KEY", "test-anon-key")
os.environ.setdefault("SUPABASE_SERVICE_ROLE_KEY", "test-service-key")
os.environ.setdefault("SUPABASE_JWT_SECRET", "test-jwt-secret")

from core.agents.pipeline.stateless.coordinator.execution import ExecutionEngine

try:
    from core.agents.pipeline.stateless.context.archiver import ContextArchiver
    HAS_ARCHIVER = True
except ImportError:
    HAS_ARCHIVER = False


# --- Helpers ---

def make_user(content, **kw):
    msg = {"role": "user", "content": content}
    msg.update(kw)
    return msg

def make_assistant(content, tool_calls=None, **kw):
    msg = {"role": "assistant", "content": content}
    if tool_calls:
        msg["tool_calls"] = tool_calls
    msg.update(kw)
    return msg

def make_tool(content, tool_call_id=None, **kw):
    msg = {"role": "tool", "content": content, "tool_call_id": tool_call_id or f"call_{uuid.uuid4().hex[:12]}"}
    msg.update(kw)
    return msg

def make_tool_call(name="some_tool"):
    return {"id": f"call_{uuid.uuid4().hex[:12]}", "type": "function", "function": {"name": name, "arguments": "{}"}}

def big_content(n):
    base = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
    return (base * ((n // len(base)) + 1))[:n]


def _mock_sandbox():
    """Create a fake sandbox that stores files in memory."""
    files = {}

    sandbox = MagicMock()

    async def upload_file(content, path):
        if isinstance(content, bytes):
            files[path] = content
        else:
            files[path] = content.encode("utf-8") if isinstance(content, str) else content

    async def download_file(path):
        if path in files:
            return files[path]
        raise FileNotFoundError(path)

    async def create_folder(path, mode="755"):
        pass  # no-op

    sandbox.fs.upload_file = upload_file
    sandbox.fs.download_file = download_file
    sandbox.fs.create_folder = create_folder

    return sandbox, files


def _mock_llm_summary_response():
    """Return a fake ModelResponse with a valid summary JSON."""
    import litellm

    summary_json = json.dumps({
        "summary": "User requested building a website. Agent searched for best practices and processed large pasted content including documentation and design specs.",
        "topics": ["web development", "website building"],
        "key_decisions": [{"decision": "Use React for frontend", "rationale": "User preference"}],
        "facts": {
            "project_name": "user-website",
            "tech_stack": ["React", "Node.js"],
            "current_goal": "Build a complete website"
        }
    })

    return litellm.completion(
        model="gpt-3.5-turbo",
        messages=[{"role": "user", "content": "test"}],
        mock_response=summary_json,
    )


# ============================================================================
# 1. Context > 200k β†’ actual summary generated β†’ tokens come down
# ============================================================================

@pytest.mark.skipif(not HAS_ARCHIVER, reason="ContextArchiver not available (main branch)")
class TestContextExceeds200k:
    """When context hits 200k tokens, compression should:
    1. Call the archiver which calls the LLM to generate a real summary
    2. Write archive files to sandbox
    3. Compress working memory (truncate giant messages)
    4. Return compressed messages with summary

    Only the LLM call and sandbox filesystem are mocked β€” everything else
    runs for real.
    """

    @pytest.mark.asyncio
    async def test_over_200k_generates_summary_and_compresses(self):
        """Full pipeline with actual summary generation (mocked LLM response)."""
        engine = ExecutionEngine(MagicMock(), MagicMock())
        engine._state.model_name = "test-model"
        engine._state.thread_id = "test-thread"
        engine._state.project_id = "test-project"
        engine._state.account_id = "test-account"
        engine._state.stream_key = "test-stream"
        engine.TEST_THRESHOLD_OVERRIDE = 100  # Force trigger

        # 10 messages with giant user pastes (the real scenario)
        tc = make_tool_call("search")
        msgs = [
            make_user("Build me a website"),
            make_assistant("Sure, let me search for best practices.", [tc]),
            make_tool(big_content(50_000), tc["id"]),
            make_assistant("Here's what I found."),
            make_user(big_content(403_000)),   # Giant paste β€” 403K chars
            make_assistant("Processing that..."),
            make_user(big_content(315_000)),   # Another giant paste β€” 315K chars
            make_assistant("Got it."),
            make_user("Now finish it up"),
            make_assistant("Done!"),
        ]
        system = {"role": "system", "content": "You are helpful."}

        # Mock sandbox filesystem (in-memory)
        sandbox, files = _mock_sandbox()
        mock_sandbox_info = MagicMock()
        mock_sandbox_info.sandbox = sandbox

        # Mock LLM to return a proper summary JSON
        mock_llm_response = _mock_llm_summary_response()

        async def mock_make_llm_call(**kwargs):
            return mock_llm_response

        async def mock_get_db_client():
            return MagicMock()
        mock_db_conn = MagicMock()
        mock_db_conn.client = mock_get_db_client()

        with patch("core.ai_models.model_manager.get_context_window", return_value=200_000), \
             patch("core.services.supabase.DBConnection", return_value=mock_db_conn), \
             patch("core.sandbox.resolver.resolve_sandbox", new_callable=AsyncMock, return_value=mock_sandbox_info), \
             patch("core.services.llm.make_llm_api_call", side_effect=mock_make_llm_call), \
             patch("core.cache.runtime_cache.set_cached_message_history", new_callable=AsyncMock), \
             patch("core.agents.pipeline.ux_streaming.stream_summarizing", new_callable=AsyncMock):

            result_msgs, _, did_compress = await engine._check_and_compress_if_needed(
                msgs, tokens=200_000, system_prompt=system
            )

        # --- Compression triggered ---
        assert did_compress is True
        assert len(result_msgs) < len(msgs), "Should have fewer messages after compression"

        # --- Summary is first message and contains actual LLM-generated content ---
        summary_msg = result_msgs[0]
        assert summary_msg.get("_is_summary_inline") is True
        assert "website" in summary_msg["content"].lower(), \
            "Summary should contain LLM-generated content about the website task"

        # --- Giant user messages in working memory are truncated ---
        for m in result_msgs:
            if m["role"] == "user" and not m.get("_is_summary_inline"):
                assert len(m["content"]) <= 5000, \
                    f"User message still {len(m['content'])} chars β€” should be truncated to ~4K"

        # --- Giant tool outputs are truncated ---
        for m in result_msgs:
            if m["role"] == "tool":
                assert len(m["content"]) <= 4000, \
                    f"Tool output still {len(m['content'])} chars β€” should be truncated"

        # --- Archive files were written to sandbox ---
        assert any("batch_001.md" in path for path in files), \
            f"Summary batch file not written. Files: {list(files.keys())}"
        assert any("manifest.json" in path for path in files), \
            f"Manifest not written. Files: {list(files.keys())}"

        # --- Summary file contains the LLM-generated summary ---
        summary_file = [v for k, v in files.items() if "batch_001.md" in k][0]
        summary_content = summary_file.decode("utf-8") if isinstance(summary_file, bytes) else summary_file
        assert "website" in summary_content.lower(), \
            "Archive summary file should contain LLM-generated summary about the website"


# ============================================================================
# 2. Orphan tool outputs β†’ don't crash the agent run
# ============================================================================

class TestOrphanToolDontCrashAgent:
    """When messages have orphan tool results (no matching assistant tool_call),
    execute_step should repair them and proceed without crashing.

    This happens when context is reloaded from DB after compression β€” the
    assistant that called the tool was archived, but the tool result wasn't.
    """

    @pytest.mark.asyncio
    async def test_orphan_tool_handled_in_execute_step(self):
        """execute_step detects orphan, repairs it, calls LLM successfully."""
        from core.agents.pipeline.stateless.context.manager import ContextManager

        state = MagicMock()
        state.model_name = "test-model"
        state.thread_id = "test-thread"
        state.stream_key = "test-stream"
        state.tool_schemas = [{"type": "function", "function": {"name": "test", "parameters": {}}}]
        state.system_prompt = {"role": "system", "content": "You are helpful."}

        # Orphan tool result β€” its matching assistant was archived
        orphan_messages = [
            make_tool("result from archived assistant call", "call_archived_001"),
            make_user("continue working on X"),
            make_assistant("Sure, working on X."),
        ]
        state.get_messages.return_value = orphan_messages
        state._messages = list(orphan_messages)

        mock_layers = MagicMock()
        mock_layers.to_messages.return_value = orphan_messages
        mock_layers.total_messages = len(orphan_messages)

        async def mock_process_response(response):
            yield {"type": "text", "content": "done"}

        processor = MagicMock()
        processor.process_response = mock_process_response

        engine = ExecutionEngine(state, processor)

        async def mock_llm_stream():
            yield {"choices": [{"delta": {"content": "hello"}}]}

        mock_executor = MagicMock()
        mock_executor.execute = AsyncMock(return_value=mock_llm_stream())

        with patch.object(ContextManager, "extract_layers", return_value=mock_layers), \
             patch.object(engine, "fast_token_count", new_callable=AsyncMock, return_value=500), \
             patch("core.agents.pipeline.ux_streaming.stream_context_usage", new_callable=AsyncMock), \
             patch.object(engine, "_check_and_compress_if_needed", new_callable=AsyncMock, return_value=(orphan_messages, 500, False)), \
             patch("core.agents.pipeline.stateless.coordinator.execution.add_cache_control", side_effect=lambda x: x), \
             patch("core.agents.pipeline.stateless.coordinator.execution.LLMExecutor", return_value=mock_executor):

            chunks = []
            async for chunk in engine.execute_step():
                chunks.append(chunk)

        # Agent should NOT have crashed
        error_chunks = [c for c in chunks if isinstance(c, dict) and c.get("type") == "error"]
        assert len(error_chunks) == 0, f"Agent crashed with errors: {error_chunks}"

        # LLM should have been called
        mock_executor.execute.assert_called_once()

        # No orphan tool results sent to LLM
        sent_messages = mock_executor.execute.call_args.kwargs["prepared_messages"]
        for msg in sent_messages:
            if msg.get("role") == "tool":
                tid = msg["tool_call_id"]
                has_parent = any(
                    tid in [t["id"] for t in m.get("tool_calls", [])]
                    for m in sent_messages if m.get("role") == "assistant"
                )
                assert has_parent, f"Orphan tool {tid} sent to LLM β€” would cause API error"