auto / tests /test_phase8_runtime_feedback.py
rsnarsna
feat(phase8): complete runtime feedback edge cases and adversarial hardening
c8c9359
Raw History Blame Contribute Delete
26.9 kB
"""
ALLI Phase 8 Test Suite: Unified Runtime Event & Feedback Telemetry
Tests the unified runtime event contract, deterministic sequence ordering,
strict credential/token scrubbing, bounded event retention, cursor pagination,
multi-tenant BOLA protection, error attribution, and FastAPI endpoints.
"""
import time
import pytest
from fastapi.testclient import TestClient
from main import app
from language_core.runtime_events import (
GLOBAL_EVENT_STORE,
RuntimeEvent,
RuntimeEventStore,
redact_sensitive_text,
redact_sensitive_data,
EVENT_RUN_STARTED,
EVENT_RUN_COMPLETED,
EVENT_RUN_FAILED,
EVENT_RUN_CANCELLED,
EVENT_NODE_QUEUED,
EVENT_NODE_STARTED,
EVENT_NODE_COMPLETED,
EVENT_NODE_FAILED,
EVENT_NODE_SKIPPED,
EVENT_NODE_CANCELLED,
EVENT_STDOUT_CHUNK,
EVENT_NODE_OUTPUT,
EVENT_TIMEOUT,
EVENT_QUOTA_EXCEEDED,
EVENT_VALIDATION_ERROR,
MAX_EVENTS_PER_RUN,
)
from language_core.workflow_executor import (
execute_workflow_graph,
execute_workflow_step,
cancel_workflow_run,
)
import language_core.workflow_executor as we
client = TestClient(app)
# ─── 1. EVENT CONTRACT & ORDERING ─────────────────────────────────────────────
def test_runtime_event_contract_and_monotonic_ordering():
"""Validates that events emitted during execution follow the versioned contract and strict monotonic seq order."""
run_id = f"test_run_ordering_{int(time.time()*1000)}"
graph = {
"run_id": run_id,
"name": "Canonical 3-Node Telemetry Test",
"nodes": [
{
"id": "inp_1",
"type": "constant.int",
"data": {"label": "Number 49", "parameters": {"literal_value": 49, "const_type": "int"}},
},
{
"id": "math_1",
"type": "math.sqrt",
"data": {"label": "Square Root", "parameters": {"rounding": "none"}},
},
{
"id": "out_1",
"type": "io.print",
"data": {"label": "Print Output", "parameters": {"prefix": "Result: "}},
},
],
"edges": [
{"id": "e1", "source": "inp_1", "target": "math_1", "source_handle": "out", "target_handle": "x"},
{"id": "e2", "source": "math_1", "target": "out_1", "source_handle": "result", "target_handle": "value"},
],
}
result = execute_workflow_graph(graph)
assert result["success"] is True, f"Workflow failed: {result.get('error')}"
assert result["run_id"] == run_id
assert "graph_revision" in result
assert len(result["graph_revision"]) == 64 # SHA-256 hex digest
events, cursor, has_more = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=100)
assert len(events) >= 6
assert has_more is False
# Check monotonic sequencing
seqs = [e.seq for e in events]
assert seqs == list(range(1, len(events) + 1)), "Sequence numbers must be strictly 1, 2, 3..."
event_types = [e.event_type for e in events]
assert event_types[0] == EVENT_RUN_STARTED
assert EVENT_NODE_QUEUED in event_types
assert EVENT_NODE_STARTED in event_types
assert EVENT_NODE_OUTPUT in event_types
assert EVENT_NODE_COMPLETED in event_types
assert event_types[-1] == EVENT_RUN_COMPLETED
# Verify contract schema properties
for e in events:
assert e.version == "1.0.0"
assert e.run_id == run_id
assert e.graph_revision == result["graph_revision"]
assert isinstance(e.timestamp, str)
assert e.severity in ("info", "warning", "error")
# ─── 2. CREDENTIAL & TOKEN REDACTION ──────────────────────────────────────────
def test_credential_redaction_patterns():
"""Ensures secrets, AWS keys, JWTs, DB passwords, and tokens are scrubbed from telemetry."""
raw_text = (
"Connected to postgresql://postgres:SuperSecretPassword123@db.supabase.co:5432/postgres. "
"AWS key AKIAIOSFODNN7EXAMPLE with session aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'. "
"Supabase sbp_1234567890abcdef1234567890. API secret_key='super_private_secret_string_123'."
)
clean = redact_sensitive_text(raw_text)
assert "SuperSecretPassword123" not in clean
assert "[REDACTED_PASSWORD]" in clean
assert "AKIAIOSFODNN7EXAMPLE" not in clean
assert "[REDACTED_AWS_KEY]" in clean
assert "wJalrXUtnFEMI" not in clean
assert "[REDACTED_SUPABASE_TOKEN]" in clean
assert "super_private_secret_string_123" not in clean
# Bearer token scrubbing
bearer_text = "Authorization: Bearer my_opaque_bearer_token_1234567890"
clean_bearer = redact_sensitive_text(bearer_text)
assert "[REDACTED_BEARER_TOKEN]" in clean_bearer
assert "my_opaque_bearer_token" not in clean_bearer
# JWT token scrubbing
jwt_text = "Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
clean_jwt = redact_sensitive_text(jwt_text)
assert "[REDACTED_JWT]" in clean_jwt
assert "eyJhbGci" not in clean_jwt
# Recursive dictionary redaction test
nested = {
"url": "mongodb+srv://admin:my_mongo_pass@cluster0.mongodb.net/production",
"nested": {
"token": "Bearer abcdef1234567890xyz",
"safe_val": 42,
},
}
scrubbed = redact_sensitive_data(nested)
assert "my_mongo_pass" not in str(scrubbed)
assert "[REDACTED_PASSWORD]" in scrubbed["url"]
assert "[REDACTED_BEARER_TOKEN]" in scrubbed["nested"]["token"]
assert scrubbed["nested"]["safe_val"] == 42
# ─── 3. BOUNDED MEMORY RING BUFFER ───────────────────────────────────────────
def test_bounded_memory_ring_buffer_enforcement():
"""Verifies that the event store caps stored events at 1000 per run without memory leaks."""
custom_store = RuntimeEventStore(max_events_per_run=100)
test_run = "run_bounded_test"
rev = "a" * 64
# Inject 250 events
for i in range(1, 251):
custom_store.record_event(
run_id=test_run,
graph_revision=rev,
event_type=EVENT_NODE_STARTED,
message=f"Event {i}",
)
# Sequence counter must strictly track total events emitted
assert custom_store._seq_counters[test_run] == 250
# In-memory event list must be strictly bounded to max_events
stored_events = custom_store._events[test_run]
assert len(stored_events) == 100
# The oldest events (1-150) should have been pruned; remaining are 151-250
assert stored_events[0].seq == 151
assert stored_events[-1].seq == 250
# ─── 4. INCREMENTAL CURSOR RETRIEVAL ─────────────────────────────────────────
def test_cursor_pagination_and_incremental_fetching():
"""Verifies cursor-based pagination returns exactly the next slice of events without duplicates."""
custom_store = RuntimeEventStore()
test_run = "run_cursor_test"
rev = "b" * 64
for i in range(1, 11):
custom_store.record_event(run_id=test_run, graph_revision=rev, event_type=EVENT_NODE_STARTED, message=f"Step {i}")
# Page 1: since_seq=0, limit=4
p1, c1, more1 = custom_store.get_events(test_run, since_seq=0, limit=4)
assert len(p1) == 4
assert [e.seq for e in p1] == [1, 2, 3, 4]
assert c1 == 4
assert more1 is True
# Page 2: since_seq=c1, limit=4
p2, c2, more2 = custom_store.get_events(test_run, since_seq=c1, limit=4)
assert len(p2) == 4
assert [e.seq for e in p2] == [5, 6, 7, 8]
assert c2 == 8
assert more2 is True
# Page 3: since_seq=c2, limit=4
p3, c3, more3 = custom_store.get_events(test_run, since_seq=c2, limit=4)
assert len(p3) == 2
assert [e.seq for e in p3] == [9, 10]
assert c3 == 10
assert more3 is False
# Page 4: since_seq=c3, limit=4 (no more events)
p4, c4, more4 = custom_store.get_events(test_run, since_seq=c3, limit=4)
assert len(p4) == 0
assert c4 == 10
assert more4 is False
# ─── 5. MULTI-TENANT BOLA ISOLATION ───────────────────────────────────────────
def test_multi_tenant_events_access_isolation():
"""Verifies that unauthorized tenants are blocked from retrieving other users' execution telemetry."""
run_id = f"run_tenant_isolated_{int(time.time()*1000)}"
rev = "c" * 64
# Record event owned by user_alice
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_RUN_STARTED,
message="Alice workflow started",
owner_user_id="user_alice_uuid",
)
meta = GLOBAL_EVENT_STORE.get_run_metadata(run_id)
assert meta["owner_user_id"] == "user_alice_uuid"
resp = client.get(f"/api/graph/events/{run_id}")
assert resp.status_code == 200
data = resp.json()
assert data["run_id"] == run_id
assert len(data["events"]) >= 1
# ─── 6. PRE-FLIGHT VALIDATION REJECTION EVENTS ────────────────────────────────
def test_preflight_validation_rejection_events():
"""Ensures pre-flight validation rejections emit validation_error diagnostics and run_failed event."""
invalid_run_id = f"run_invalid_preflight_{int(time.time()*1000)}"
invalid_graph = {
"run_id": invalid_run_id,
"nodes": [
{
"id": "node_dup",
"type": "input",
"data": {"value": 10},
},
{
"id": "node_dup", # Duplicate ID violates DAG schema
"type": "input",
"data": {"value": 20},
},
],
"edges": [],
}
result = execute_workflow_graph(invalid_graph)
assert result["success"] is False
assert result["status"] == "failed"
events, _, _ = GLOBAL_EVENT_STORE.get_events(invalid_run_id, since_seq=0, limit=50)
assert len(events) >= 2
types = [e.event_type for e in events]
assert EVENT_VALIDATION_ERROR in types
assert EVENT_RUN_FAILED in types
# ─── 7. TIMEOUT EVENT EMISSION ────────────────────────────────────────────────
def test_timeout_event_emission_and_node_cascade():
"""Verifies that timeouts emit timeout and run_failed events while marking downstream nodes skipped."""
timeout_run_id = f"run_timeout_{int(time.time()*1000)}"
graph = {
"run_id": timeout_run_id,
"timeout_seconds": 0.05,
"nodes": [
{
"id": "n1",
"type": "primitive.variable",
"data": {"type": "primitive.variable", "value": 10, "label": "Node 1"},
},
{
"id": "n2",
"type": "primitive.variable",
"data": {"type": "primitive.variable", "value": 20, "label": "Node 2"},
},
],
"edges": [
{"id": "e1", "source": "n1", "target": "n2", "sourceHandle": "val", "targetHandle": "in_val"},
],
}
orig_handler = we._execute_variable_node
def slow_variable_node(inputs, params):
time.sleep(0.1)
return orig_handler(inputs, params)
try:
we._execute_variable_node = slow_variable_node
result = execute_workflow_graph(graph, timeout_seconds=0.05)
assert result["success"] is False
events, _, _ = GLOBAL_EVENT_STORE.get_events(timeout_run_id, since_seq=0, limit=50)
types = [e.event_type for e in events]
assert EVENT_TIMEOUT in types
assert EVENT_RUN_FAILED in types
finally:
we._execute_variable_node = orig_handler
# ─── 8. CANCELLATION TELEMETRY ────────────────────────────────────────────────
def test_cancellation_telemetry_events():
"""Verifies that cancel_workflow_run emits node_cancelled and run_cancelled events."""
cancel_run_id = f"run_cancel_test_{int(time.time()*1000)}"
rev = "d" * 64
GLOBAL_EVENT_STORE.record_event(
run_id=cancel_run_id,
graph_revision=rev,
event_type=EVENT_RUN_STARTED,
message="Workflow started",
)
node_results = {
"node_1": {"node_id": "node_1", "status": "running", "exec_state": "running"},
"node_2": {"node_id": "node_2", "status": "queued", "exec_state": "queued"},
}
res = cancel_workflow_run(run_id=cancel_run_id, node_results=node_results)
assert res["status"] == "cancelled"
assert "node_1" in res["cancelled_nodes"]
assert "node_2" in res["cancelled_nodes"]
events, _, _ = GLOBAL_EVENT_STORE.get_events(cancel_run_id, since_seq=0, limit=50)
types = [e.event_type for e in events]
assert EVENT_NODE_CANCELLED in types
assert EVENT_RUN_CANCELLED in types
# ─── 9. OUTPUT QUOTA EXCEEDED TELEMETRY ───────────────────────────────────────
def test_output_quota_exceeded_telemetry_event():
"""Verifies that output payload exceeding 1MB emits quota_exceeded event with code ERR-OUTPUT-SIZE-EXCEEDED."""
oversized_run_id = f"run_quota_exceeded_{int(time.time()*1000)}"
oversized_string = "X" * (1024 * 1024 + 100) # > 1MB
graph = {
"run_id": oversized_run_id,
"nodes": [
{
"id": "huge_node",
"type": "constant.string",
"data": {"label": "Oversized String", "parameters": {"literal_value": oversized_string, "const_type": "string"}},
},
],
"edges": [],
}
result = execute_workflow_graph(graph)
assert result["success"] is False
events, _, _ = GLOBAL_EVENT_STORE.get_events(oversized_run_id, since_seq=0, limit=50)
types = [e.event_type for e in events]
assert EVENT_QUOTA_EXCEEDED in types
quota_events = [e for e in events if e.event_type == EVENT_QUOTA_EXCEEDED]
assert len(quota_events) >= 1
assert quota_events[0].code == "ERR-OUTPUT-SIZE-EXCEEDED"
# ─── 10. FASTAPI TELEMETRY ENDPOINTS INTEGRATION ──────────────────────────────
def test_fastapi_runtime_feedback_endpoints():
"""Tests the /graph/events/{run_id} and /graph/feedback/{run_id} endpoints."""
run_id = f"run_api_test_{int(time.time()*1000)}"
rev = "e" * 64
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_RUN_STARTED,
message="API run started",
)
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_NODE_STARTED,
node_id="n1",
message="Executing node n1",
)
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_RUN_COMPLETED,
message="API run completed",
)
# 1. Test /graph/events
ev_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=10")
assert ev_resp.status_code == 200
ev_data = ev_resp.json()
assert ev_data["run_id"] == run_id
assert ev_data["graph_revision"] == rev
assert len(ev_data["events"]) == 3
assert ev_data["cursor"] == 3
assert ev_data["is_complete"] is True
# 2. Test /graph/feedback snapshot
fb_resp = client.get(f"/api/graph/feedback/{run_id}")
assert fb_resp.status_code == 200
fb_data = fb_resp.json()
assert fb_data["run_id"] == run_id
assert fb_data["graph_revision"] == rev
assert fb_data["lifecycle_status"] == "success"
assert fb_data["is_complete"] is True
# 3. Test invalid run_id identifier rejected
malformed_resp = client.get("/api/graph/events/../../etc/passwd")
assert malformed_resp.status_code in (400, 404)
# ─── 11. EDGE CASE 1: EVENT RETENTION BUFFER RECOVERY & GAP SIGNALING ─────────
def test_edge_case_ring_buffer_retention_and_gap_recovery():
"""
Verifies ring buffer behavior when events exceed MAX_EVENTS_PER_RUN (1,000).
When client cursor is older than the oldest retained event, event store returns
events_truncated=True, oldest_retained_seq, and endpoint issues warning.
"""
run_id = f"run_retention_gap_{int(time.time()*1000)}"
rev = "f" * 64
# Generate 1,050 events (exceeding 1,000-event ring buffer)
for i in range(1050):
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_NODE_STARTED if i % 2 == 0 else EVENT_NODE_COMPLETED,
node_id=f"node_{i}",
message=f"Event {i}",
)
# Client requests from cursor 0 (which was pruned by ring buffer)
res = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=50)
assert res.events_truncated is True
assert res.oldest_retained_seq > 0
assert len(res[0]) > 0
# First returned event matches oldest_retained_seq
assert res[0][0].seq == res.oldest_retained_seq
# Test API endpoint response includes warning and retention metadata
api_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=50")
assert api_resp.status_code == 200
data = api_resp.json()
assert data["events_truncated"] is True
assert data["oldest_retained_seq"] == res.oldest_retained_seq
assert "warning" in data
assert "truncated" in data["warning"]
# ─── 12. EDGE CASE 2: ADVANCED SECRETS & NESTED REDACTION ────────────────────
def test_edge_case_advanced_credentials_and_nested_redaction():
"""
Verifies sanitization of:
- URL-encoded passwords in connection strings (e.g. mysql://user:p%40ssw0rd@host:3306)
- HTTP Basic Auth headers (Basic dXNlcjpzZWNyZXQ=)
- Generic multiline secret assignments (client_secret, webhook_secret, password)
- Deeply nested structures (dicts inside lists inside dicts)
"""
from language_core.runtime_events import scrub_sensitive_string, sanitize_event_payload
# 1. URL-encoded passwords in connection URIs
uri_text = "Connecting to postgresql://db_user:MyP%40ssw0rd%21@db.internal.net:5432/production_db"
scrubbed_uri = scrub_sensitive_string(uri_text)
assert "MyP%40ssw0rd%21" not in scrubbed_uri
assert "postgresql://db_user:[REDACTED_PASSWORD]@db.internal.net:5432/production_db" in scrubbed_uri
# 2. HTTP Basic Auth header
basic_header = "Authorization: Basic dXNlcjpwYXNzd29yZDEyMw=="
scrubbed_basic = scrub_sensitive_string(basic_header)
assert "Basic [REDACTED_BASIC_AUTH]" in scrubbed_basic
assert "dXNlcjpwYXNzd29yZDEyMw==" not in scrubbed_basic
# 3. Multiline secret assignments
multiline_code = (
'def setup_keys():\n'
' client_secret = "cs_live_998877665544332211"\n'
' webhook_secret = "whsec_abcdef1234567890"\n'
' password = "SuperSecretPassword!"\n'
' return True'
)
scrubbed_multi = scrub_sensitive_string(multiline_code)
assert "cs_live_998877665544332211" not in scrubbed_multi
assert "whsec_abcdef1234567890" not in scrubbed_multi
assert "SuperSecretPassword!" not in scrubbed_multi
assert "[REDACTED]" in scrubbed_multi
# 4. Deeply nested payloads
nested_payload = {
"config": {
"services": [
{
"name": "payment_gateway",
"dsn": "postgres://pay_admin:TopSecretPass123@sql.prod:5432/payments",
"tokens": {"api_key": "ak_live_xyz_9876543210"},
}
]
}
}
sanitized = sanitize_event_payload(nested_payload)
dsn_clean = sanitized["config"]["services"][0]["dsn"]
assert "TopSecretPass123" not in dsn_clean
assert "[REDACTED_PASSWORD]" in dsn_clean
assert sanitized["config"]["services"][0]["tokens"]["api_key"] == "[REDACTED]"
# ─── 13. EDGE CASE 3: MULTI-TENANT AUTHORIZATION & BOLA/IDOR DEFENSE ─────────
def test_edge_case_multi_tenant_access_isolation():
"""
Verifies that Tenant B is strictly forbidden (HTTP 403 ERR-FORBIDDEN-SESSION)
from accessing Tenant A's:
1. /graph/events/{run_id}
2. /graph/feedback/{run_id}
3. /graph/events/{run_id}/stream
While Tenant A and Admin can access them.
"""
run_id = f"run_bola_tenant_a_{int(time.time()*1000)}"
tenant_a_id = "tenant_alice_uuid_111"
tenant_b_id = "tenant_bob_uuid_222"
rev = "a" * 64
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_RUN_STARTED,
message="Tenant A execution started",
owner_user_id=tenant_a_id,
)
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_RUN_COMPLETED,
message="Tenant A execution completed",
owner_user_id=tenant_a_id,
)
# 1. Tenant B attempts /graph/events -> 403 Forbidden
res_b_events = client.get(
f"/api/graph/events/{run_id}",
headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"}
)
assert res_b_events.status_code == 403
assert res_b_events.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION"
# 2. Tenant B attempts /graph/feedback -> 403 Forbidden
res_b_fb = client.get(
f"/api/graph/feedback/{run_id}",
headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"}
)
assert res_b_fb.status_code == 403
assert res_b_fb.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION"
# 3. Tenant B attempts /graph/events/{run_id}/stream -> 403 Forbidden
res_b_stream = client.get(
f"/api/graph/events/{run_id}/stream",
headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"}
)
assert res_b_stream.status_code == 403
assert res_b_stream.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION"
# 4. Tenant A accesses their own run -> 200 OK
res_a = client.get(
f"/api/graph/events/{run_id}",
headers={"x-user-id": tenant_a_id, "x-user-role": "authenticated"}
)
assert res_a.status_code == 200
assert len(res_a.json()["events"]) >= 2
# 5. Admin accesses Tenant A's run -> 200 OK
res_admin = client.get(
f"/api/graph/events/{run_id}",
headers={"x-user-id": "service_account", "x-user-role": "admin"}
)
assert res_admin.status_code == 200
# ─── 14. EDGE CASE 4: OUTPUT QUOTA CASCADE ISOLATION ─────────────────────────
def test_edge_case_output_quota_cascade_prevents_partial_propagation():
"""
Verifies that when an upstream node fails the 1 MB output quota (ERR-OUTPUT-SIZE-EXCEEDED),
the downstream node is SKIPPED with WARN-DEPENDENCY-FAILED and NEVER receives
a truncated preview as an actual execution input.
"""
from language_core.workflow_executor import execute_workflow_graph
run_id = f"run_quota_cascade_{int(time.time()*1000)}"
# Node 1 produces oversized string (>1MB quota)
# Node 2 depends on Node 1's output
graph = {
"run_id": run_id,
"nodes": [
{
"id": "node_oversized",
"type": "input",
"data": {"value": "X" * 1_050_000},
},
{
"id": "node_downstream",
"type": "output",
"data": {},
},
],
"edges": [
{
"id": "e_oversized_downstream",
"source": "node_oversized",
"target": "node_downstream",
"sourceHandle": "output",
"targetHandle": "input",
}
],
}
results = execute_workflow_graph(graph, timeout_seconds=5.0)
assert results["success"] is False
assert results["status"] == "failed"
# Node 1 must be failed with ERR-OUTPUT-SIZE-EXCEEDED
oversized_res = results["node_results"]["node_oversized"]
assert oversized_res["status"] == "failed"
assert "ERR-OUTPUT-SIZE-EXCEEDED" in oversized_res["error"]
assert oversized_res.get("error_semantics", {}).get("code") == "ERR-OUTPUT-SIZE-EXCEEDED"
# Node 2 must be skipped with WARN-DEPENDENCY-FAILED and NO inputs consumed
downstream_res = results["node_results"]["node_downstream"]
assert downstream_res["status"] == "skipped"
assert "Upstream dependency" in downstream_res["error"]
assert downstream_res.get("error_semantics", {}).get("code") == "WARN-DEPENDENCY-FAILED"
assert downstream_res["inputs"] == {}
# ─── 15. EDGE CASE 5: SSE STREAM RECONNECTION & CURSOR RECOVERY ──────────────
def test_edge_case_sse_streaming_reconnection_and_terminal_packet():
"""
Verifies SSE stream reconnection behavior:
1. Resumption via Last-Event-ID header
2. Resumption via cursor query parameter
3. Proper SSE event packaging (id:, event:, data:)
4. Terminal stream_end event emission upon completion
"""
run_id = f"run_sse_recon_{int(time.time()*1000)}"
rev = "b" * 64
# Record 5 events
for i in range(1, 6):
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=rev,
event_type=EVENT_NODE_STARTED if i < 5 else EVENT_RUN_COMPLETED,
node_id=f"node_{i}",
message=f"Step {i}",
)
# Reconnect using Last-Event-ID = 3 (expect events 4 and 5, then stream_end)
with client.stream("GET", f"/api/graph/events/{run_id}/stream", headers={"last-event-id": "3"}) as response:
assert response.status_code == 200
assert "text/event-stream" in response.headers["content-type"]
lines = []
for line in response.iter_lines():
if line:
lines.append(line)
if "stream_end" in line:
break
stream_output = "\n".join(lines)
# Check that events 4 and 5 are received with valid id tags
assert "id: 4" in stream_output or "node_4" in stream_output
assert "id: 5" in stream_output or "node_5" in stream_output
# Check terminal packet
assert "event: stream_end" in stream_output