""" ALLI Phase 8 Test Suite: Unified Runtime Event & Feedback Telemetry Tests the unified runtime event contract, deterministic sequence ordering, strict credential/token scrubbing, bounded event retention, cursor pagination, multi-tenant BOLA protection, error attribution, and FastAPI endpoints. """ import time import pytest from fastapi.testclient import TestClient from main import app from language_core.runtime_events import ( GLOBAL_EVENT_STORE, RuntimeEvent, RuntimeEventStore, redact_sensitive_text, redact_sensitive_data, EVENT_RUN_STARTED, EVENT_RUN_COMPLETED, EVENT_RUN_FAILED, EVENT_RUN_CANCELLED, EVENT_NODE_QUEUED, EVENT_NODE_STARTED, EVENT_NODE_COMPLETED, EVENT_NODE_FAILED, EVENT_NODE_SKIPPED, EVENT_NODE_CANCELLED, EVENT_STDOUT_CHUNK, EVENT_NODE_OUTPUT, EVENT_TIMEOUT, EVENT_QUOTA_EXCEEDED, EVENT_VALIDATION_ERROR, MAX_EVENTS_PER_RUN, ) from language_core.workflow_executor import ( execute_workflow_graph, execute_workflow_step, cancel_workflow_run, ) import language_core.workflow_executor as we client = TestClient(app) # ─── 1. EVENT CONTRACT & ORDERING ───────────────────────────────────────────── def test_runtime_event_contract_and_monotonic_ordering(): """Validates that events emitted during execution follow the versioned contract and strict monotonic seq order.""" run_id = f"test_run_ordering_{int(time.time()*1000)}" graph = { "run_id": run_id, "name": "Canonical 3-Node Telemetry Test", "nodes": [ { "id": "inp_1", "type": "constant.int", "data": {"label": "Number 49", "parameters": {"literal_value": 49, "const_type": "int"}}, }, { "id": "math_1", "type": "math.sqrt", "data": {"label": "Square Root", "parameters": {"rounding": "none"}}, }, { "id": "out_1", "type": "io.print", "data": {"label": "Print Output", "parameters": {"prefix": "Result: "}}, }, ], "edges": [ {"id": "e1", "source": "inp_1", "target": "math_1", "source_handle": "out", "target_handle": "x"}, {"id": "e2", "source": "math_1", "target": "out_1", "source_handle": "result", "target_handle": "value"}, ], } result = execute_workflow_graph(graph) assert result["success"] is True, f"Workflow failed: {result.get('error')}" assert result["run_id"] == run_id assert "graph_revision" in result assert len(result["graph_revision"]) == 64 # SHA-256 hex digest events, cursor, has_more = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=100) assert len(events) >= 6 assert has_more is False # Check monotonic sequencing seqs = [e.seq for e in events] assert seqs == list(range(1, len(events) + 1)), "Sequence numbers must be strictly 1, 2, 3..." event_types = [e.event_type for e in events] assert event_types[0] == EVENT_RUN_STARTED assert EVENT_NODE_QUEUED in event_types assert EVENT_NODE_STARTED in event_types assert EVENT_NODE_OUTPUT in event_types assert EVENT_NODE_COMPLETED in event_types assert event_types[-1] == EVENT_RUN_COMPLETED # Verify contract schema properties for e in events: assert e.version == "1.0.0" assert e.run_id == run_id assert e.graph_revision == result["graph_revision"] assert isinstance(e.timestamp, str) assert e.severity in ("info", "warning", "error") # ─── 2. CREDENTIAL & TOKEN REDACTION ────────────────────────────────────────── def test_credential_redaction_patterns(): """Ensures secrets, AWS keys, JWTs, DB passwords, and tokens are scrubbed from telemetry.""" raw_text = ( "Connected to postgresql://postgres:SuperSecretPassword123@db.supabase.co:5432/postgres. " "AWS key AKIAIOSFODNN7EXAMPLE with session aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'. " "Supabase sbp_1234567890abcdef1234567890. API secret_key='super_private_secret_string_123'." ) clean = redact_sensitive_text(raw_text) assert "SuperSecretPassword123" not in clean assert "[REDACTED_PASSWORD]" in clean assert "AKIAIOSFODNN7EXAMPLE" not in clean assert "[REDACTED_AWS_KEY]" in clean assert "wJalrXUtnFEMI" not in clean assert "[REDACTED_SUPABASE_TOKEN]" in clean assert "super_private_secret_string_123" not in clean # Bearer token scrubbing bearer_text = "Authorization: Bearer my_opaque_bearer_token_1234567890" clean_bearer = redact_sensitive_text(bearer_text) assert "[REDACTED_BEARER_TOKEN]" in clean_bearer assert "my_opaque_bearer_token" not in clean_bearer # JWT token scrubbing jwt_text = "Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" clean_jwt = redact_sensitive_text(jwt_text) assert "[REDACTED_JWT]" in clean_jwt assert "eyJhbGci" not in clean_jwt # Recursive dictionary redaction test nested = { "url": "mongodb+srv://admin:my_mongo_pass@cluster0.mongodb.net/production", "nested": { "token": "Bearer abcdef1234567890xyz", "safe_val": 42, }, } scrubbed = redact_sensitive_data(nested) assert "my_mongo_pass" not in str(scrubbed) assert "[REDACTED_PASSWORD]" in scrubbed["url"] assert "[REDACTED_BEARER_TOKEN]" in scrubbed["nested"]["token"] assert scrubbed["nested"]["safe_val"] == 42 # ─── 3. BOUNDED MEMORY RING BUFFER ─────────────────────────────────────────── def test_bounded_memory_ring_buffer_enforcement(): """Verifies that the event store caps stored events at 1000 per run without memory leaks.""" custom_store = RuntimeEventStore(max_events_per_run=100) test_run = "run_bounded_test" rev = "a" * 64 # Inject 250 events for i in range(1, 251): custom_store.record_event( run_id=test_run, graph_revision=rev, event_type=EVENT_NODE_STARTED, message=f"Event {i}", ) # Sequence counter must strictly track total events emitted assert custom_store._seq_counters[test_run] == 250 # In-memory event list must be strictly bounded to max_events stored_events = custom_store._events[test_run] assert len(stored_events) == 100 # The oldest events (1-150) should have been pruned; remaining are 151-250 assert stored_events[0].seq == 151 assert stored_events[-1].seq == 250 # ─── 4. INCREMENTAL CURSOR RETRIEVAL ───────────────────────────────────────── def test_cursor_pagination_and_incremental_fetching(): """Verifies cursor-based pagination returns exactly the next slice of events without duplicates.""" custom_store = RuntimeEventStore() test_run = "run_cursor_test" rev = "b" * 64 for i in range(1, 11): custom_store.record_event(run_id=test_run, graph_revision=rev, event_type=EVENT_NODE_STARTED, message=f"Step {i}") # Page 1: since_seq=0, limit=4 p1, c1, more1 = custom_store.get_events(test_run, since_seq=0, limit=4) assert len(p1) == 4 assert [e.seq for e in p1] == [1, 2, 3, 4] assert c1 == 4 assert more1 is True # Page 2: since_seq=c1, limit=4 p2, c2, more2 = custom_store.get_events(test_run, since_seq=c1, limit=4) assert len(p2) == 4 assert [e.seq for e in p2] == [5, 6, 7, 8] assert c2 == 8 assert more2 is True # Page 3: since_seq=c2, limit=4 p3, c3, more3 = custom_store.get_events(test_run, since_seq=c2, limit=4) assert len(p3) == 2 assert [e.seq for e in p3] == [9, 10] assert c3 == 10 assert more3 is False # Page 4: since_seq=c3, limit=4 (no more events) p4, c4, more4 = custom_store.get_events(test_run, since_seq=c3, limit=4) assert len(p4) == 0 assert c4 == 10 assert more4 is False # ─── 5. MULTI-TENANT BOLA ISOLATION ─────────────────────────────────────────── def test_multi_tenant_events_access_isolation(): """Verifies that unauthorized tenants are blocked from retrieving other users' execution telemetry.""" run_id = f"run_tenant_isolated_{int(time.time()*1000)}" rev = "c" * 64 # Record event owned by user_alice GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_RUN_STARTED, message="Alice workflow started", owner_user_id="user_alice_uuid", ) meta = GLOBAL_EVENT_STORE.get_run_metadata(run_id) assert meta["owner_user_id"] == "user_alice_uuid" resp = client.get(f"/api/graph/events/{run_id}") assert resp.status_code == 200 data = resp.json() assert data["run_id"] == run_id assert len(data["events"]) >= 1 # ─── 6. PRE-FLIGHT VALIDATION REJECTION EVENTS ──────────────────────────────── def test_preflight_validation_rejection_events(): """Ensures pre-flight validation rejections emit validation_error diagnostics and run_failed event.""" invalid_run_id = f"run_invalid_preflight_{int(time.time()*1000)}" invalid_graph = { "run_id": invalid_run_id, "nodes": [ { "id": "node_dup", "type": "input", "data": {"value": 10}, }, { "id": "node_dup", # Duplicate ID violates DAG schema "type": "input", "data": {"value": 20}, }, ], "edges": [], } result = execute_workflow_graph(invalid_graph) assert result["success"] is False assert result["status"] == "failed" events, _, _ = GLOBAL_EVENT_STORE.get_events(invalid_run_id, since_seq=0, limit=50) assert len(events) >= 2 types = [e.event_type for e in events] assert EVENT_VALIDATION_ERROR in types assert EVENT_RUN_FAILED in types # ─── 7. TIMEOUT EVENT EMISSION ──────────────────────────────────────────────── def test_timeout_event_emission_and_node_cascade(): """Verifies that timeouts emit timeout and run_failed events while marking downstream nodes skipped.""" timeout_run_id = f"run_timeout_{int(time.time()*1000)}" graph = { "run_id": timeout_run_id, "timeout_seconds": 0.05, "nodes": [ { "id": "n1", "type": "primitive.variable", "data": {"type": "primitive.variable", "value": 10, "label": "Node 1"}, }, { "id": "n2", "type": "primitive.variable", "data": {"type": "primitive.variable", "value": 20, "label": "Node 2"}, }, ], "edges": [ {"id": "e1", "source": "n1", "target": "n2", "sourceHandle": "val", "targetHandle": "in_val"}, ], } orig_handler = we._execute_variable_node def slow_variable_node(inputs, params): time.sleep(0.1) return orig_handler(inputs, params) try: we._execute_variable_node = slow_variable_node result = execute_workflow_graph(graph, timeout_seconds=0.05) assert result["success"] is False events, _, _ = GLOBAL_EVENT_STORE.get_events(timeout_run_id, since_seq=0, limit=50) types = [e.event_type for e in events] assert EVENT_TIMEOUT in types assert EVENT_RUN_FAILED in types finally: we._execute_variable_node = orig_handler # ─── 8. CANCELLATION TELEMETRY ──────────────────────────────────────────────── def test_cancellation_telemetry_events(): """Verifies that cancel_workflow_run emits node_cancelled and run_cancelled events.""" cancel_run_id = f"run_cancel_test_{int(time.time()*1000)}" rev = "d" * 64 GLOBAL_EVENT_STORE.record_event( run_id=cancel_run_id, graph_revision=rev, event_type=EVENT_RUN_STARTED, message="Workflow started", ) node_results = { "node_1": {"node_id": "node_1", "status": "running", "exec_state": "running"}, "node_2": {"node_id": "node_2", "status": "queued", "exec_state": "queued"}, } res = cancel_workflow_run(run_id=cancel_run_id, node_results=node_results) assert res["status"] == "cancelled" assert "node_1" in res["cancelled_nodes"] assert "node_2" in res["cancelled_nodes"] events, _, _ = GLOBAL_EVENT_STORE.get_events(cancel_run_id, since_seq=0, limit=50) types = [e.event_type for e in events] assert EVENT_NODE_CANCELLED in types assert EVENT_RUN_CANCELLED in types # ─── 9. OUTPUT QUOTA EXCEEDED TELEMETRY ─────────────────────────────────────── def test_output_quota_exceeded_telemetry_event(): """Verifies that output payload exceeding 1MB emits quota_exceeded event with code ERR-OUTPUT-SIZE-EXCEEDED.""" oversized_run_id = f"run_quota_exceeded_{int(time.time()*1000)}" oversized_string = "X" * (1024 * 1024 + 100) # > 1MB graph = { "run_id": oversized_run_id, "nodes": [ { "id": "huge_node", "type": "constant.string", "data": {"label": "Oversized String", "parameters": {"literal_value": oversized_string, "const_type": "string"}}, }, ], "edges": [], } result = execute_workflow_graph(graph) assert result["success"] is False events, _, _ = GLOBAL_EVENT_STORE.get_events(oversized_run_id, since_seq=0, limit=50) types = [e.event_type for e in events] assert EVENT_QUOTA_EXCEEDED in types quota_events = [e for e in events if e.event_type == EVENT_QUOTA_EXCEEDED] assert len(quota_events) >= 1 assert quota_events[0].code == "ERR-OUTPUT-SIZE-EXCEEDED" # ─── 10. FASTAPI TELEMETRY ENDPOINTS INTEGRATION ────────────────────────────── def test_fastapi_runtime_feedback_endpoints(): """Tests the /graph/events/{run_id} and /graph/feedback/{run_id} endpoints.""" run_id = f"run_api_test_{int(time.time()*1000)}" rev = "e" * 64 GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_RUN_STARTED, message="API run started", ) GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_NODE_STARTED, node_id="n1", message="Executing node n1", ) GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_RUN_COMPLETED, message="API run completed", ) # 1. Test /graph/events ev_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=10") assert ev_resp.status_code == 200 ev_data = ev_resp.json() assert ev_data["run_id"] == run_id assert ev_data["graph_revision"] == rev assert len(ev_data["events"]) == 3 assert ev_data["cursor"] == 3 assert ev_data["is_complete"] is True # 2. Test /graph/feedback snapshot fb_resp = client.get(f"/api/graph/feedback/{run_id}") assert fb_resp.status_code == 200 fb_data = fb_resp.json() assert fb_data["run_id"] == run_id assert fb_data["graph_revision"] == rev assert fb_data["lifecycle_status"] == "success" assert fb_data["is_complete"] is True # 3. Test invalid run_id identifier rejected malformed_resp = client.get("/api/graph/events/../../etc/passwd") assert malformed_resp.status_code in (400, 404) # ─── 11. EDGE CASE 1: EVENT RETENTION BUFFER RECOVERY & GAP SIGNALING ───────── def test_edge_case_ring_buffer_retention_and_gap_recovery(): """ Verifies ring buffer behavior when events exceed MAX_EVENTS_PER_RUN (1,000). When client cursor is older than the oldest retained event, event store returns events_truncated=True, oldest_retained_seq, and endpoint issues warning. """ run_id = f"run_retention_gap_{int(time.time()*1000)}" rev = "f" * 64 # Generate 1,050 events (exceeding 1,000-event ring buffer) for i in range(1050): GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_NODE_STARTED if i % 2 == 0 else EVENT_NODE_COMPLETED, node_id=f"node_{i}", message=f"Event {i}", ) # Client requests from cursor 0 (which was pruned by ring buffer) res = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=50) assert res.events_truncated is True assert res.oldest_retained_seq > 0 assert len(res[0]) > 0 # First returned event matches oldest_retained_seq assert res[0][0].seq == res.oldest_retained_seq # Test API endpoint response includes warning and retention metadata api_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=50") assert api_resp.status_code == 200 data = api_resp.json() assert data["events_truncated"] is True assert data["oldest_retained_seq"] == res.oldest_retained_seq assert "warning" in data assert "truncated" in data["warning"] # ─── 12. EDGE CASE 2: ADVANCED SECRETS & NESTED REDACTION ──────────────────── def test_edge_case_advanced_credentials_and_nested_redaction(): """ Verifies sanitization of: - URL-encoded passwords in connection strings (e.g. mysql://user:p%40ssw0rd@host:3306) - HTTP Basic Auth headers (Basic dXNlcjpzZWNyZXQ=) - Generic multiline secret assignments (client_secret, webhook_secret, password) - Deeply nested structures (dicts inside lists inside dicts) """ from language_core.runtime_events import scrub_sensitive_string, sanitize_event_payload # 1. URL-encoded passwords in connection URIs uri_text = "Connecting to postgresql://db_user:MyP%40ssw0rd%21@db.internal.net:5432/production_db" scrubbed_uri = scrub_sensitive_string(uri_text) assert "MyP%40ssw0rd%21" not in scrubbed_uri assert "postgresql://db_user:[REDACTED_PASSWORD]@db.internal.net:5432/production_db" in scrubbed_uri # 2. HTTP Basic Auth header basic_header = "Authorization: Basic dXNlcjpwYXNzd29yZDEyMw==" scrubbed_basic = scrub_sensitive_string(basic_header) assert "Basic [REDACTED_BASIC_AUTH]" in scrubbed_basic assert "dXNlcjpwYXNzd29yZDEyMw==" not in scrubbed_basic # 3. Multiline secret assignments multiline_code = ( 'def setup_keys():\n' ' client_secret = "cs_live_998877665544332211"\n' ' webhook_secret = "whsec_abcdef1234567890"\n' ' password = "SuperSecretPassword!"\n' ' return True' ) scrubbed_multi = scrub_sensitive_string(multiline_code) assert "cs_live_998877665544332211" not in scrubbed_multi assert "whsec_abcdef1234567890" not in scrubbed_multi assert "SuperSecretPassword!" not in scrubbed_multi assert "[REDACTED]" in scrubbed_multi # 4. Deeply nested payloads nested_payload = { "config": { "services": [ { "name": "payment_gateway", "dsn": "postgres://pay_admin:TopSecretPass123@sql.prod:5432/payments", "tokens": {"api_key": "ak_live_xyz_9876543210"}, } ] } } sanitized = sanitize_event_payload(nested_payload) dsn_clean = sanitized["config"]["services"][0]["dsn"] assert "TopSecretPass123" not in dsn_clean assert "[REDACTED_PASSWORD]" in dsn_clean assert sanitized["config"]["services"][0]["tokens"]["api_key"] == "[REDACTED]" # ─── 13. EDGE CASE 3: MULTI-TENANT AUTHORIZATION & BOLA/IDOR DEFENSE ───────── def test_edge_case_multi_tenant_access_isolation(): """ Verifies that Tenant B is strictly forbidden (HTTP 403 ERR-FORBIDDEN-SESSION) from accessing Tenant A's: 1. /graph/events/{run_id} 2. /graph/feedback/{run_id} 3. /graph/events/{run_id}/stream While Tenant A and Admin can access them. """ run_id = f"run_bola_tenant_a_{int(time.time()*1000)}" tenant_a_id = "tenant_alice_uuid_111" tenant_b_id = "tenant_bob_uuid_222" rev = "a" * 64 GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_RUN_STARTED, message="Tenant A execution started", owner_user_id=tenant_a_id, ) GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_RUN_COMPLETED, message="Tenant A execution completed", owner_user_id=tenant_a_id, ) # 1. Tenant B attempts /graph/events -> 403 Forbidden res_b_events = client.get( f"/api/graph/events/{run_id}", headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"} ) assert res_b_events.status_code == 403 assert res_b_events.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION" # 2. Tenant B attempts /graph/feedback -> 403 Forbidden res_b_fb = client.get( f"/api/graph/feedback/{run_id}", headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"} ) assert res_b_fb.status_code == 403 assert res_b_fb.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION" # 3. Tenant B attempts /graph/events/{run_id}/stream -> 403 Forbidden res_b_stream = client.get( f"/api/graph/events/{run_id}/stream", headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"} ) assert res_b_stream.status_code == 403 assert res_b_stream.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION" # 4. Tenant A accesses their own run -> 200 OK res_a = client.get( f"/api/graph/events/{run_id}", headers={"x-user-id": tenant_a_id, "x-user-role": "authenticated"} ) assert res_a.status_code == 200 assert len(res_a.json()["events"]) >= 2 # 5. Admin accesses Tenant A's run -> 200 OK res_admin = client.get( f"/api/graph/events/{run_id}", headers={"x-user-id": "service_account", "x-user-role": "admin"} ) assert res_admin.status_code == 200 # ─── 14. EDGE CASE 4: OUTPUT QUOTA CASCADE ISOLATION ───────────────────────── def test_edge_case_output_quota_cascade_prevents_partial_propagation(): """ Verifies that when an upstream node fails the 1 MB output quota (ERR-OUTPUT-SIZE-EXCEEDED), the downstream node is SKIPPED with WARN-DEPENDENCY-FAILED and NEVER receives a truncated preview as an actual execution input. """ from language_core.workflow_executor import execute_workflow_graph run_id = f"run_quota_cascade_{int(time.time()*1000)}" # Node 1 produces oversized string (>1MB quota) # Node 2 depends on Node 1's output graph = { "run_id": run_id, "nodes": [ { "id": "node_oversized", "type": "input", "data": {"value": "X" * 1_050_000}, }, { "id": "node_downstream", "type": "output", "data": {}, }, ], "edges": [ { "id": "e_oversized_downstream", "source": "node_oversized", "target": "node_downstream", "sourceHandle": "output", "targetHandle": "input", } ], } results = execute_workflow_graph(graph, timeout_seconds=5.0) assert results["success"] is False assert results["status"] == "failed" # Node 1 must be failed with ERR-OUTPUT-SIZE-EXCEEDED oversized_res = results["node_results"]["node_oversized"] assert oversized_res["status"] == "failed" assert "ERR-OUTPUT-SIZE-EXCEEDED" in oversized_res["error"] assert oversized_res.get("error_semantics", {}).get("code") == "ERR-OUTPUT-SIZE-EXCEEDED" # Node 2 must be skipped with WARN-DEPENDENCY-FAILED and NO inputs consumed downstream_res = results["node_results"]["node_downstream"] assert downstream_res["status"] == "skipped" assert "Upstream dependency" in downstream_res["error"] assert downstream_res.get("error_semantics", {}).get("code") == "WARN-DEPENDENCY-FAILED" assert downstream_res["inputs"] == {} # ─── 15. EDGE CASE 5: SSE STREAM RECONNECTION & CURSOR RECOVERY ────────────── def test_edge_case_sse_streaming_reconnection_and_terminal_packet(): """ Verifies SSE stream reconnection behavior: 1. Resumption via Last-Event-ID header 2. Resumption via cursor query parameter 3. Proper SSE event packaging (id:, event:, data:) 4. Terminal stream_end event emission upon completion """ run_id = f"run_sse_recon_{int(time.time()*1000)}" rev = "b" * 64 # Record 5 events for i in range(1, 6): GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=rev, event_type=EVENT_NODE_STARTED if i < 5 else EVENT_RUN_COMPLETED, node_id=f"node_{i}", message=f"Step {i}", ) # Reconnect using Last-Event-ID = 3 (expect events 4 and 5, then stream_end) with client.stream("GET", f"/api/graph/events/{run_id}/stream", headers={"last-event-id": "3"}) as response: assert response.status_code == 200 assert "text/event-stream" in response.headers["content-type"] lines = [] for line in response.iter_lines(): if line: lines.append(line) if "stream_end" in line: break stream_output = "\n".join(lines) # Check that events 4 and 5 are received with valid id tags assert "id: 4" in stream_output or "node_4" in stream_output assert "id: 5" in stream_output or "node_5" in stream_output # Check terminal packet assert "event: stream_end" in stream_output