File size: 26,881 Bytes
284800d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c8c9359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
"""
ALLI Phase 8 Test Suite: Unified Runtime Event & Feedback Telemetry
Tests the unified runtime event contract, deterministic sequence ordering,
strict credential/token scrubbing, bounded event retention, cursor pagination,
multi-tenant BOLA protection, error attribution, and FastAPI endpoints.
"""

import time
import pytest
from fastapi.testclient import TestClient

from main import app
from language_core.runtime_events import (
    GLOBAL_EVENT_STORE,
    RuntimeEvent,
    RuntimeEventStore,
    redact_sensitive_text,
    redact_sensitive_data,
    EVENT_RUN_STARTED,
    EVENT_RUN_COMPLETED,
    EVENT_RUN_FAILED,
    EVENT_RUN_CANCELLED,
    EVENT_NODE_QUEUED,
    EVENT_NODE_STARTED,
    EVENT_NODE_COMPLETED,
    EVENT_NODE_FAILED,
    EVENT_NODE_SKIPPED,
    EVENT_NODE_CANCELLED,
    EVENT_STDOUT_CHUNK,
    EVENT_NODE_OUTPUT,
    EVENT_TIMEOUT,
    EVENT_QUOTA_EXCEEDED,
    EVENT_VALIDATION_ERROR,
    MAX_EVENTS_PER_RUN,
)
from language_core.workflow_executor import (
    execute_workflow_graph,
    execute_workflow_step,
    cancel_workflow_run,
)
import language_core.workflow_executor as we

client = TestClient(app)


# ─── 1. EVENT CONTRACT & ORDERING ─────────────────────────────────────────────

def test_runtime_event_contract_and_monotonic_ordering():
    """Validates that events emitted during execution follow the versioned contract and strict monotonic seq order."""
    run_id = f"test_run_ordering_{int(time.time()*1000)}"
    graph = {
        "run_id": run_id,
        "name": "Canonical 3-Node Telemetry Test",
        "nodes": [
            {
                "id": "inp_1",
                "type": "constant.int",
                "data": {"label": "Number 49", "parameters": {"literal_value": 49, "const_type": "int"}},
            },
            {
                "id": "math_1",
                "type": "math.sqrt",
                "data": {"label": "Square Root", "parameters": {"rounding": "none"}},
            },
            {
                "id": "out_1",
                "type": "io.print",
                "data": {"label": "Print Output", "parameters": {"prefix": "Result: "}},
            },
        ],
        "edges": [
            {"id": "e1", "source": "inp_1", "target": "math_1", "source_handle": "out", "target_handle": "x"},
            {"id": "e2", "source": "math_1", "target": "out_1", "source_handle": "result", "target_handle": "value"},
        ],
    }

    result = execute_workflow_graph(graph)
    assert result["success"] is True, f"Workflow failed: {result.get('error')}"
    assert result["run_id"] == run_id
    assert "graph_revision" in result
    assert len(result["graph_revision"]) == 64  # SHA-256 hex digest

    events, cursor, has_more = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=100)
    assert len(events) >= 6
    assert has_more is False

    # Check monotonic sequencing
    seqs = [e.seq for e in events]
    assert seqs == list(range(1, len(events) + 1)), "Sequence numbers must be strictly 1, 2, 3..."

    event_types = [e.event_type for e in events]
    assert event_types[0] == EVENT_RUN_STARTED
    assert EVENT_NODE_QUEUED in event_types
    assert EVENT_NODE_STARTED in event_types
    assert EVENT_NODE_OUTPUT in event_types
    assert EVENT_NODE_COMPLETED in event_types
    assert event_types[-1] == EVENT_RUN_COMPLETED

    # Verify contract schema properties
    for e in events:
        assert e.version == "1.0.0"
        assert e.run_id == run_id
        assert e.graph_revision == result["graph_revision"]
        assert isinstance(e.timestamp, str)
        assert e.severity in ("info", "warning", "error")


# ─── 2. CREDENTIAL & TOKEN REDACTION ──────────────────────────────────────────

def test_credential_redaction_patterns():
    """Ensures secrets, AWS keys, JWTs, DB passwords, and tokens are scrubbed from telemetry."""
    raw_text = (
        "Connected to postgresql://postgres:SuperSecretPassword123@db.supabase.co:5432/postgres. "
        "AWS key AKIAIOSFODNN7EXAMPLE with session aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'. "
        "Supabase sbp_1234567890abcdef1234567890. API secret_key='super_private_secret_string_123'."
    )

    clean = redact_sensitive_text(raw_text)

    assert "SuperSecretPassword123" not in clean
    assert "[REDACTED_PASSWORD]" in clean
    assert "AKIAIOSFODNN7EXAMPLE" not in clean
    assert "[REDACTED_AWS_KEY]" in clean
    assert "wJalrXUtnFEMI" not in clean
    assert "[REDACTED_SUPABASE_TOKEN]" in clean
    assert "super_private_secret_string_123" not in clean

    # Bearer token scrubbing
    bearer_text = "Authorization: Bearer my_opaque_bearer_token_1234567890"
    clean_bearer = redact_sensitive_text(bearer_text)
    assert "[REDACTED_BEARER_TOKEN]" in clean_bearer
    assert "my_opaque_bearer_token" not in clean_bearer

    # JWT token scrubbing
    jwt_text = "Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
    clean_jwt = redact_sensitive_text(jwt_text)
    assert "[REDACTED_JWT]" in clean_jwt
    assert "eyJhbGci" not in clean_jwt

    # Recursive dictionary redaction test
    nested = {
        "url": "mongodb+srv://admin:my_mongo_pass@cluster0.mongodb.net/production",
        "nested": {
            "token": "Bearer abcdef1234567890xyz",
            "safe_val": 42,
        },
    }
    scrubbed = redact_sensitive_data(nested)
    assert "my_mongo_pass" not in str(scrubbed)
    assert "[REDACTED_PASSWORD]" in scrubbed["url"]
    assert "[REDACTED_BEARER_TOKEN]" in scrubbed["nested"]["token"]
    assert scrubbed["nested"]["safe_val"] == 42


# ─── 3. BOUNDED MEMORY RING BUFFER ───────────────────────────────────────────

def test_bounded_memory_ring_buffer_enforcement():
    """Verifies that the event store caps stored events at 1000 per run without memory leaks."""
    custom_store = RuntimeEventStore(max_events_per_run=100)
    test_run = "run_bounded_test"
    rev = "a" * 64

    # Inject 250 events
    for i in range(1, 251):
        custom_store.record_event(
            run_id=test_run,
            graph_revision=rev,
            event_type=EVENT_NODE_STARTED,
            message=f"Event {i}",
        )

    # Sequence counter must strictly track total events emitted
    assert custom_store._seq_counters[test_run] == 250

    # In-memory event list must be strictly bounded to max_events
    stored_events = custom_store._events[test_run]
    assert len(stored_events) == 100

    # The oldest events (1-150) should have been pruned; remaining are 151-250
    assert stored_events[0].seq == 151
    assert stored_events[-1].seq == 250


# ─── 4. INCREMENTAL CURSOR RETRIEVAL ─────────────────────────────────────────

def test_cursor_pagination_and_incremental_fetching():
    """Verifies cursor-based pagination returns exactly the next slice of events without duplicates."""
    custom_store = RuntimeEventStore()
    test_run = "run_cursor_test"
    rev = "b" * 64

    for i in range(1, 11):
        custom_store.record_event(run_id=test_run, graph_revision=rev, event_type=EVENT_NODE_STARTED, message=f"Step {i}")

    # Page 1: since_seq=0, limit=4
    p1, c1, more1 = custom_store.get_events(test_run, since_seq=0, limit=4)
    assert len(p1) == 4
    assert [e.seq for e in p1] == [1, 2, 3, 4]
    assert c1 == 4
    assert more1 is True

    # Page 2: since_seq=c1, limit=4
    p2, c2, more2 = custom_store.get_events(test_run, since_seq=c1, limit=4)
    assert len(p2) == 4
    assert [e.seq for e in p2] == [5, 6, 7, 8]
    assert c2 == 8
    assert more2 is True

    # Page 3: since_seq=c2, limit=4
    p3, c3, more3 = custom_store.get_events(test_run, since_seq=c2, limit=4)
    assert len(p3) == 2
    assert [e.seq for e in p3] == [9, 10]
    assert c3 == 10
    assert more3 is False

    # Page 4: since_seq=c3, limit=4 (no more events)
    p4, c4, more4 = custom_store.get_events(test_run, since_seq=c3, limit=4)
    assert len(p4) == 0
    assert c4 == 10
    assert more4 is False


# ─── 5. MULTI-TENANT BOLA ISOLATION ───────────────────────────────────────────

def test_multi_tenant_events_access_isolation():
    """Verifies that unauthorized tenants are blocked from retrieving other users' execution telemetry."""
    run_id = f"run_tenant_isolated_{int(time.time()*1000)}"
    rev = "c" * 64

    # Record event owned by user_alice
    GLOBAL_EVENT_STORE.record_event(
        run_id=run_id,
        graph_revision=rev,
        event_type=EVENT_RUN_STARTED,
        message="Alice workflow started",
        owner_user_id="user_alice_uuid",
    )

    meta = GLOBAL_EVENT_STORE.get_run_metadata(run_id)
    assert meta["owner_user_id"] == "user_alice_uuid"

    resp = client.get(f"/api/graph/events/{run_id}")
    assert resp.status_code == 200
    data = resp.json()
    assert data["run_id"] == run_id
    assert len(data["events"]) >= 1


# ─── 6. PRE-FLIGHT VALIDATION REJECTION EVENTS ────────────────────────────────

def test_preflight_validation_rejection_events():
    """Ensures pre-flight validation rejections emit validation_error diagnostics and run_failed event."""
    invalid_run_id = f"run_invalid_preflight_{int(time.time()*1000)}"
    invalid_graph = {
        "run_id": invalid_run_id,
        "nodes": [
            {
                "id": "node_dup",
                "type": "input",
                "data": {"value": 10},
            },
            {
                "id": "node_dup",  # Duplicate ID violates DAG schema
                "type": "input",
                "data": {"value": 20},
            },
        ],
        "edges": [],
    }

    result = execute_workflow_graph(invalid_graph)
    assert result["success"] is False
    assert result["status"] == "failed"

    events, _, _ = GLOBAL_EVENT_STORE.get_events(invalid_run_id, since_seq=0, limit=50)
    assert len(events) >= 2
    types = [e.event_type for e in events]
    assert EVENT_VALIDATION_ERROR in types
    assert EVENT_RUN_FAILED in types


# ─── 7. TIMEOUT EVENT EMISSION ────────────────────────────────────────────────

def test_timeout_event_emission_and_node_cascade():
    """Verifies that timeouts emit timeout and run_failed events while marking downstream nodes skipped."""
    timeout_run_id = f"run_timeout_{int(time.time()*1000)}"
    graph = {
        "run_id": timeout_run_id,
        "timeout_seconds": 0.05,
        "nodes": [
            {
                "id": "n1",
                "type": "primitive.variable",
                "data": {"type": "primitive.variable", "value": 10, "label": "Node 1"},
            },
            {
                "id": "n2",
                "type": "primitive.variable",
                "data": {"type": "primitive.variable", "value": 20, "label": "Node 2"},
            },
        ],
        "edges": [
            {"id": "e1", "source": "n1", "target": "n2", "sourceHandle": "val", "targetHandle": "in_val"},
        ],
    }

    orig_handler = we._execute_variable_node

    def slow_variable_node(inputs, params):
        time.sleep(0.1)
        return orig_handler(inputs, params)

    try:
        we._execute_variable_node = slow_variable_node
        result = execute_workflow_graph(graph, timeout_seconds=0.05)
        assert result["success"] is False

        events, _, _ = GLOBAL_EVENT_STORE.get_events(timeout_run_id, since_seq=0, limit=50)
        types = [e.event_type for e in events]
        assert EVENT_TIMEOUT in types
        assert EVENT_RUN_FAILED in types
    finally:
        we._execute_variable_node = orig_handler


# ─── 8. CANCELLATION TELEMETRY ────────────────────────────────────────────────

def test_cancellation_telemetry_events():
    """Verifies that cancel_workflow_run emits node_cancelled and run_cancelled events."""
    cancel_run_id = f"run_cancel_test_{int(time.time()*1000)}"
    rev = "d" * 64

    GLOBAL_EVENT_STORE.record_event(
        run_id=cancel_run_id,
        graph_revision=rev,
        event_type=EVENT_RUN_STARTED,
        message="Workflow started",
    )

    node_results = {
        "node_1": {"node_id": "node_1", "status": "running", "exec_state": "running"},
        "node_2": {"node_id": "node_2", "status": "queued", "exec_state": "queued"},
    }

    res = cancel_workflow_run(run_id=cancel_run_id, node_results=node_results)
    assert res["status"] == "cancelled"
    assert "node_1" in res["cancelled_nodes"]
    assert "node_2" in res["cancelled_nodes"]

    events, _, _ = GLOBAL_EVENT_STORE.get_events(cancel_run_id, since_seq=0, limit=50)
    types = [e.event_type for e in events]
    assert EVENT_NODE_CANCELLED in types
    assert EVENT_RUN_CANCELLED in types


# ─── 9. OUTPUT QUOTA EXCEEDED TELEMETRY ───────────────────────────────────────

def test_output_quota_exceeded_telemetry_event():
    """Verifies that output payload exceeding 1MB emits quota_exceeded event with code ERR-OUTPUT-SIZE-EXCEEDED."""
    oversized_run_id = f"run_quota_exceeded_{int(time.time()*1000)}"
    oversized_string = "X" * (1024 * 1024 + 100)  # > 1MB

    graph = {
        "run_id": oversized_run_id,
        "nodes": [
            {
                "id": "huge_node",
                "type": "constant.string",
                "data": {"label": "Oversized String", "parameters": {"literal_value": oversized_string, "const_type": "string"}},
            },
        ],
        "edges": [],
    }

    result = execute_workflow_graph(graph)
    assert result["success"] is False

    events, _, _ = GLOBAL_EVENT_STORE.get_events(oversized_run_id, since_seq=0, limit=50)
    types = [e.event_type for e in events]
    assert EVENT_QUOTA_EXCEEDED in types
    quota_events = [e for e in events if e.event_type == EVENT_QUOTA_EXCEEDED]
    assert len(quota_events) >= 1
    assert quota_events[0].code == "ERR-OUTPUT-SIZE-EXCEEDED"


# ─── 10. FASTAPI TELEMETRY ENDPOINTS INTEGRATION ──────────────────────────────

def test_fastapi_runtime_feedback_endpoints():
    """Tests the /graph/events/{run_id} and /graph/feedback/{run_id} endpoints."""
    run_id = f"run_api_test_{int(time.time()*1000)}"
    rev = "e" * 64

    GLOBAL_EVENT_STORE.record_event(
        run_id=run_id,
        graph_revision=rev,
        event_type=EVENT_RUN_STARTED,
        message="API run started",
    )
    GLOBAL_EVENT_STORE.record_event(
        run_id=run_id,
        graph_revision=rev,
        event_type=EVENT_NODE_STARTED,
        node_id="n1",
        message="Executing node n1",
    )
    GLOBAL_EVENT_STORE.record_event(
        run_id=run_id,
        graph_revision=rev,
        event_type=EVENT_RUN_COMPLETED,
        message="API run completed",
    )

    # 1. Test /graph/events
    ev_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=10")
    assert ev_resp.status_code == 200
    ev_data = ev_resp.json()
    assert ev_data["run_id"] == run_id
    assert ev_data["graph_revision"] == rev
    assert len(ev_data["events"]) == 3
    assert ev_data["cursor"] == 3
    assert ev_data["is_complete"] is True

    # 2. Test /graph/feedback snapshot
    fb_resp = client.get(f"/api/graph/feedback/{run_id}")
    assert fb_resp.status_code == 200
    fb_data = fb_resp.json()
    assert fb_data["run_id"] == run_id
    assert fb_data["graph_revision"] == rev
    assert fb_data["lifecycle_status"] == "success"
    assert fb_data["is_complete"] is True

    # 3. Test invalid run_id identifier rejected
    malformed_resp = client.get("/api/graph/events/../../etc/passwd")
    assert malformed_resp.status_code in (400, 404)


# ─── 11. EDGE CASE 1: EVENT RETENTION BUFFER RECOVERY & GAP SIGNALING ─────────

def test_edge_case_ring_buffer_retention_and_gap_recovery():
    """
    Verifies ring buffer behavior when events exceed MAX_EVENTS_PER_RUN (1,000).
    When client cursor is older than the oldest retained event, event store returns
    events_truncated=True, oldest_retained_seq, and endpoint issues warning.
    """
    run_id = f"run_retention_gap_{int(time.time()*1000)}"
    rev = "f" * 64

    # Generate 1,050 events (exceeding 1,000-event ring buffer)
    for i in range(1050):
        GLOBAL_EVENT_STORE.record_event(
            run_id=run_id,
            graph_revision=rev,
            event_type=EVENT_NODE_STARTED if i % 2 == 0 else EVENT_NODE_COMPLETED,
            node_id=f"node_{i}",
            message=f"Event {i}",
        )

    # Client requests from cursor 0 (which was pruned by ring buffer)
    res = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=50)
    assert res.events_truncated is True
    assert res.oldest_retained_seq > 0
    assert len(res[0]) > 0
    # First returned event matches oldest_retained_seq
    assert res[0][0].seq == res.oldest_retained_seq

    # Test API endpoint response includes warning and retention metadata
    api_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=50")
    assert api_resp.status_code == 200
    data = api_resp.json()
    assert data["events_truncated"] is True
    assert data["oldest_retained_seq"] == res.oldest_retained_seq
    assert "warning" in data
    assert "truncated" in data["warning"]


# ─── 12. EDGE CASE 2: ADVANCED SECRETS & NESTED REDACTION ────────────────────

def test_edge_case_advanced_credentials_and_nested_redaction():
    """
    Verifies sanitization of:
    - URL-encoded passwords in connection strings (e.g. mysql://user:p%40ssw0rd@host:3306)
    - HTTP Basic Auth headers (Basic dXNlcjpzZWNyZXQ=)
    - Generic multiline secret assignments (client_secret, webhook_secret, password)
    - Deeply nested structures (dicts inside lists inside dicts)
    """
    from language_core.runtime_events import scrub_sensitive_string, sanitize_event_payload

    # 1. URL-encoded passwords in connection URIs
    uri_text = "Connecting to postgresql://db_user:MyP%40ssw0rd%21@db.internal.net:5432/production_db"
    scrubbed_uri = scrub_sensitive_string(uri_text)
    assert "MyP%40ssw0rd%21" not in scrubbed_uri
    assert "postgresql://db_user:[REDACTED_PASSWORD]@db.internal.net:5432/production_db" in scrubbed_uri

    # 2. HTTP Basic Auth header
    basic_header = "Authorization: Basic dXNlcjpwYXNzd29yZDEyMw=="
    scrubbed_basic = scrub_sensitive_string(basic_header)
    assert "Basic [REDACTED_BASIC_AUTH]" in scrubbed_basic
    assert "dXNlcjpwYXNzd29yZDEyMw==" not in scrubbed_basic

    # 3. Multiline secret assignments
    multiline_code = (
        'def setup_keys():\n'
        '    client_secret = "cs_live_998877665544332211"\n'
        '    webhook_secret = "whsec_abcdef1234567890"\n'
        '    password = "SuperSecretPassword!"\n'
        '    return True'
    )
    scrubbed_multi = scrub_sensitive_string(multiline_code)
    assert "cs_live_998877665544332211" not in scrubbed_multi
    assert "whsec_abcdef1234567890" not in scrubbed_multi
    assert "SuperSecretPassword!" not in scrubbed_multi
    assert "[REDACTED]" in scrubbed_multi

    # 4. Deeply nested payloads
    nested_payload = {
        "config": {
            "services": [
                {
                    "name": "payment_gateway",
                    "dsn": "postgres://pay_admin:TopSecretPass123@sql.prod:5432/payments",
                    "tokens": {"api_key": "ak_live_xyz_9876543210"},
                }
            ]
        }
    }
    sanitized = sanitize_event_payload(nested_payload)
    dsn_clean = sanitized["config"]["services"][0]["dsn"]
    assert "TopSecretPass123" not in dsn_clean
    assert "[REDACTED_PASSWORD]" in dsn_clean
    assert sanitized["config"]["services"][0]["tokens"]["api_key"] == "[REDACTED]"


# ─── 13. EDGE CASE 3: MULTI-TENANT AUTHORIZATION & BOLA/IDOR DEFENSE ─────────

def test_edge_case_multi_tenant_access_isolation():
    """
    Verifies that Tenant B is strictly forbidden (HTTP 403 ERR-FORBIDDEN-SESSION)
    from accessing Tenant A's:
    1. /graph/events/{run_id}
    2. /graph/feedback/{run_id}
    3. /graph/events/{run_id}/stream
    While Tenant A and Admin can access them.
    """
    run_id = f"run_bola_tenant_a_{int(time.time()*1000)}"
    tenant_a_id = "tenant_alice_uuid_111"
    tenant_b_id = "tenant_bob_uuid_222"
    rev = "a" * 64

    GLOBAL_EVENT_STORE.record_event(
        run_id=run_id,
        graph_revision=rev,
        event_type=EVENT_RUN_STARTED,
        message="Tenant A execution started",
        owner_user_id=tenant_a_id,
    )
    GLOBAL_EVENT_STORE.record_event(
        run_id=run_id,
        graph_revision=rev,
        event_type=EVENT_RUN_COMPLETED,
        message="Tenant A execution completed",
        owner_user_id=tenant_a_id,
    )

    # 1. Tenant B attempts /graph/events -> 403 Forbidden
    res_b_events = client.get(
        f"/api/graph/events/{run_id}",
        headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"}
    )
    assert res_b_events.status_code == 403
    assert res_b_events.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION"

    # 2. Tenant B attempts /graph/feedback -> 403 Forbidden
    res_b_fb = client.get(
        f"/api/graph/feedback/{run_id}",
        headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"}
    )
    assert res_b_fb.status_code == 403
    assert res_b_fb.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION"

    # 3. Tenant B attempts /graph/events/{run_id}/stream -> 403 Forbidden
    res_b_stream = client.get(
        f"/api/graph/events/{run_id}/stream",
        headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"}
    )
    assert res_b_stream.status_code == 403
    assert res_b_stream.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION"

    # 4. Tenant A accesses their own run -> 200 OK
    res_a = client.get(
        f"/api/graph/events/{run_id}",
        headers={"x-user-id": tenant_a_id, "x-user-role": "authenticated"}
    )
    assert res_a.status_code == 200
    assert len(res_a.json()["events"]) >= 2

    # 5. Admin accesses Tenant A's run -> 200 OK
    res_admin = client.get(
        f"/api/graph/events/{run_id}",
        headers={"x-user-id": "service_account", "x-user-role": "admin"}
    )
    assert res_admin.status_code == 200


# ─── 14. EDGE CASE 4: OUTPUT QUOTA CASCADE ISOLATION ─────────────────────────

def test_edge_case_output_quota_cascade_prevents_partial_propagation():
    """
    Verifies that when an upstream node fails the 1 MB output quota (ERR-OUTPUT-SIZE-EXCEEDED),
    the downstream node is SKIPPED with WARN-DEPENDENCY-FAILED and NEVER receives
    a truncated preview as an actual execution input.
    """
    from language_core.workflow_executor import execute_workflow_graph

    run_id = f"run_quota_cascade_{int(time.time()*1000)}"
    # Node 1 produces oversized string (>1MB quota)
    # Node 2 depends on Node 1's output
    graph = {
        "run_id": run_id,
        "nodes": [
            {
                "id": "node_oversized",
                "type": "input",
                "data": {"value": "X" * 1_050_000},
            },
            {
                "id": "node_downstream",
                "type": "output",
                "data": {},
            },
        ],
        "edges": [
            {
                "id": "e_oversized_downstream",
                "source": "node_oversized",
                "target": "node_downstream",
                "sourceHandle": "output",
                "targetHandle": "input",
            }
        ],
    }

    results = execute_workflow_graph(graph, timeout_seconds=5.0)
    assert results["success"] is False
    assert results["status"] == "failed"

    # Node 1 must be failed with ERR-OUTPUT-SIZE-EXCEEDED
    oversized_res = results["node_results"]["node_oversized"]
    assert oversized_res["status"] == "failed"
    assert "ERR-OUTPUT-SIZE-EXCEEDED" in oversized_res["error"]
    assert oversized_res.get("error_semantics", {}).get("code") == "ERR-OUTPUT-SIZE-EXCEEDED"

    # Node 2 must be skipped with WARN-DEPENDENCY-FAILED and NO inputs consumed
    downstream_res = results["node_results"]["node_downstream"]
    assert downstream_res["status"] == "skipped"
    assert "Upstream dependency" in downstream_res["error"]
    assert downstream_res.get("error_semantics", {}).get("code") == "WARN-DEPENDENCY-FAILED"
    assert downstream_res["inputs"] == {}


# ─── 15. EDGE CASE 5: SSE STREAM RECONNECTION & CURSOR RECOVERY ──────────────

def test_edge_case_sse_streaming_reconnection_and_terminal_packet():
    """
    Verifies SSE stream reconnection behavior:
    1. Resumption via Last-Event-ID header
    2. Resumption via cursor query parameter
    3. Proper SSE event packaging (id:, event:, data:)
    4. Terminal stream_end event emission upon completion
    """
    run_id = f"run_sse_recon_{int(time.time()*1000)}"
    rev = "b" * 64

    # Record 5 events
    for i in range(1, 6):
        GLOBAL_EVENT_STORE.record_event(
            run_id=run_id,
            graph_revision=rev,
            event_type=EVENT_NODE_STARTED if i < 5 else EVENT_RUN_COMPLETED,
            node_id=f"node_{i}",
            message=f"Step {i}",
        )

    # Reconnect using Last-Event-ID = 3 (expect events 4 and 5, then stream_end)
    with client.stream("GET", f"/api/graph/events/{run_id}/stream", headers={"last-event-id": "3"}) as response:
        assert response.status_code == 200
        assert "text/event-stream" in response.headers["content-type"]
        
        lines = []
        for line in response.iter_lines():
            if line:
                lines.append(line)
            if "stream_end" in line:
                break

    stream_output = "\n".join(lines)
    # Check that events 4 and 5 are received with valid id tags
    assert "id: 4" in stream_output or "node_4" in stream_output
    assert "id: 5" in stream_output or "node_5" in stream_output
    # Check terminal packet
    assert "event: stream_end" in stream_output