Download tests/test_phase8_runtime_feedback.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 26.9 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_phase8_runtime_feedback.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/tests/test_phase8_runtime_feedback.py
-
curl -L -o test_phase8_runtime_feedback.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_phase8_runtime_feedback.py
26.9 kB
| """ | |
| ALLI Phase 8 Test Suite: Unified Runtime Event & Feedback Telemetry | |
| Tests the unified runtime event contract, deterministic sequence ordering, | |
| strict credential/token scrubbing, bounded event retention, cursor pagination, | |
| multi-tenant BOLA protection, error attribution, and FastAPI endpoints. | |
| """ | |
| import time | |
| import pytest | |
| from fastapi.testclient import TestClient | |
| from main import app | |
| from language_core.runtime_events import ( | |
| GLOBAL_EVENT_STORE, | |
| RuntimeEvent, | |
| RuntimeEventStore, | |
| redact_sensitive_text, | |
| redact_sensitive_data, | |
| EVENT_RUN_STARTED, | |
| EVENT_RUN_COMPLETED, | |
| EVENT_RUN_FAILED, | |
| EVENT_RUN_CANCELLED, | |
| EVENT_NODE_QUEUED, | |
| EVENT_NODE_STARTED, | |
| EVENT_NODE_COMPLETED, | |
| EVENT_NODE_FAILED, | |
| EVENT_NODE_SKIPPED, | |
| EVENT_NODE_CANCELLED, | |
| EVENT_STDOUT_CHUNK, | |
| EVENT_NODE_OUTPUT, | |
| EVENT_TIMEOUT, | |
| EVENT_QUOTA_EXCEEDED, | |
| EVENT_VALIDATION_ERROR, | |
| MAX_EVENTS_PER_RUN, | |
| ) | |
| from language_core.workflow_executor import ( | |
| execute_workflow_graph, | |
| execute_workflow_step, | |
| cancel_workflow_run, | |
| ) | |
| import language_core.workflow_executor as we | |
| client = TestClient(app) | |
| # ─── 1. EVENT CONTRACT & ORDERING ───────────────────────────────────────────── | |
| def test_runtime_event_contract_and_monotonic_ordering(): | |
| """Validates that events emitted during execution follow the versioned contract and strict monotonic seq order.""" | |
| run_id = f"test_run_ordering_{int(time.time()*1000)}" | |
| graph = { | |
| "run_id": run_id, | |
| "name": "Canonical 3-Node Telemetry Test", | |
| "nodes": [ | |
| { | |
| "id": "inp_1", | |
| "type": "constant.int", | |
| "data": {"label": "Number 49", "parameters": {"literal_value": 49, "const_type": "int"}}, | |
| }, | |
| { | |
| "id": "math_1", | |
| "type": "math.sqrt", | |
| "data": {"label": "Square Root", "parameters": {"rounding": "none"}}, | |
| }, | |
| { | |
| "id": "out_1", | |
| "type": "io.print", | |
| "data": {"label": "Print Output", "parameters": {"prefix": "Result: "}}, | |
| }, | |
| ], | |
| "edges": [ | |
| {"id": "e1", "source": "inp_1", "target": "math_1", "source_handle": "out", "target_handle": "x"}, | |
| {"id": "e2", "source": "math_1", "target": "out_1", "source_handle": "result", "target_handle": "value"}, | |
| ], | |
| } | |
| result = execute_workflow_graph(graph) | |
| assert result["success"] is True, f"Workflow failed: {result.get('error')}" | |
| assert result["run_id"] == run_id | |
| assert "graph_revision" in result | |
| assert len(result["graph_revision"]) == 64 # SHA-256 hex digest | |
| events, cursor, has_more = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=100) | |
| assert len(events) >= 6 | |
| assert has_more is False | |
| # Check monotonic sequencing | |
| seqs = [e.seq for e in events] | |
| assert seqs == list(range(1, len(events) + 1)), "Sequence numbers must be strictly 1, 2, 3..." | |
| event_types = [e.event_type for e in events] | |
| assert event_types[0] == EVENT_RUN_STARTED | |
| assert EVENT_NODE_QUEUED in event_types | |
| assert EVENT_NODE_STARTED in event_types | |
| assert EVENT_NODE_OUTPUT in event_types | |
| assert EVENT_NODE_COMPLETED in event_types | |
| assert event_types[-1] == EVENT_RUN_COMPLETED | |
| # Verify contract schema properties | |
| for e in events: | |
| assert e.version == "1.0.0" | |
| assert e.run_id == run_id | |
| assert e.graph_revision == result["graph_revision"] | |
| assert isinstance(e.timestamp, str) | |
| assert e.severity in ("info", "warning", "error") | |
| # ─── 2. CREDENTIAL & TOKEN REDACTION ────────────────────────────────────────── | |
| def test_credential_redaction_patterns(): | |
| """Ensures secrets, AWS keys, JWTs, DB passwords, and tokens are scrubbed from telemetry.""" | |
| raw_text = ( | |
| "Connected to postgresql://postgres:SuperSecretPassword123@db.supabase.co:5432/postgres. " | |
| "AWS key AKIAIOSFODNN7EXAMPLE with session aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'. " | |
| "Supabase sbp_1234567890abcdef1234567890. API secret_key='super_private_secret_string_123'." | |
| ) | |
| clean = redact_sensitive_text(raw_text) | |
| assert "SuperSecretPassword123" not in clean | |
| assert "[REDACTED_PASSWORD]" in clean | |
| assert "AKIAIOSFODNN7EXAMPLE" not in clean | |
| assert "[REDACTED_AWS_KEY]" in clean | |
| assert "wJalrXUtnFEMI" not in clean | |
| assert "[REDACTED_SUPABASE_TOKEN]" in clean | |
| assert "super_private_secret_string_123" not in clean | |
| # Bearer token scrubbing | |
| bearer_text = "Authorization: Bearer my_opaque_bearer_token_1234567890" | |
| clean_bearer = redact_sensitive_text(bearer_text) | |
| assert "[REDACTED_BEARER_TOKEN]" in clean_bearer | |
| assert "my_opaque_bearer_token" not in clean_bearer | |
| # JWT token scrubbing | |
| jwt_text = "Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" | |
| clean_jwt = redact_sensitive_text(jwt_text) | |
| assert "[REDACTED_JWT]" in clean_jwt | |
| assert "eyJhbGci" not in clean_jwt | |
| # Recursive dictionary redaction test | |
| nested = { | |
| "url": "mongodb+srv://admin:my_mongo_pass@cluster0.mongodb.net/production", | |
| "nested": { | |
| "token": "Bearer abcdef1234567890xyz", | |
| "safe_val": 42, | |
| }, | |
| } | |
| scrubbed = redact_sensitive_data(nested) | |
| assert "my_mongo_pass" not in str(scrubbed) | |
| assert "[REDACTED_PASSWORD]" in scrubbed["url"] | |
| assert "[REDACTED_BEARER_TOKEN]" in scrubbed["nested"]["token"] | |
| assert scrubbed["nested"]["safe_val"] == 42 | |
| # ─── 3. BOUNDED MEMORY RING BUFFER ─────────────────────────────────────────── | |
| def test_bounded_memory_ring_buffer_enforcement(): | |
| """Verifies that the event store caps stored events at 1000 per run without memory leaks.""" | |
| custom_store = RuntimeEventStore(max_events_per_run=100) | |
| test_run = "run_bounded_test" | |
| rev = "a" * 64 | |
| # Inject 250 events | |
| for i in range(1, 251): | |
| custom_store.record_event( | |
| run_id=test_run, | |
| graph_revision=rev, | |
| event_type=EVENT_NODE_STARTED, | |
| message=f"Event {i}", | |
| ) | |
| # Sequence counter must strictly track total events emitted | |
| assert custom_store._seq_counters[test_run] == 250 | |
| # In-memory event list must be strictly bounded to max_events | |
| stored_events = custom_store._events[test_run] | |
| assert len(stored_events) == 100 | |
| # The oldest events (1-150) should have been pruned; remaining are 151-250 | |
| assert stored_events[0].seq == 151 | |
| assert stored_events[-1].seq == 250 | |
| # ─── 4. INCREMENTAL CURSOR RETRIEVAL ───────────────────────────────────────── | |
| def test_cursor_pagination_and_incremental_fetching(): | |
| """Verifies cursor-based pagination returns exactly the next slice of events without duplicates.""" | |
| custom_store = RuntimeEventStore() | |
| test_run = "run_cursor_test" | |
| rev = "b" * 64 | |
| for i in range(1, 11): | |
| custom_store.record_event(run_id=test_run, graph_revision=rev, event_type=EVENT_NODE_STARTED, message=f"Step {i}") | |
| # Page 1: since_seq=0, limit=4 | |
| p1, c1, more1 = custom_store.get_events(test_run, since_seq=0, limit=4) | |
| assert len(p1) == 4 | |
| assert [e.seq for e in p1] == [1, 2, 3, 4] | |
| assert c1 == 4 | |
| assert more1 is True | |
| # Page 2: since_seq=c1, limit=4 | |
| p2, c2, more2 = custom_store.get_events(test_run, since_seq=c1, limit=4) | |
| assert len(p2) == 4 | |
| assert [e.seq for e in p2] == [5, 6, 7, 8] | |
| assert c2 == 8 | |
| assert more2 is True | |
| # Page 3: since_seq=c2, limit=4 | |
| p3, c3, more3 = custom_store.get_events(test_run, since_seq=c2, limit=4) | |
| assert len(p3) == 2 | |
| assert [e.seq for e in p3] == [9, 10] | |
| assert c3 == 10 | |
| assert more3 is False | |
| # Page 4: since_seq=c3, limit=4 (no more events) | |
| p4, c4, more4 = custom_store.get_events(test_run, since_seq=c3, limit=4) | |
| assert len(p4) == 0 | |
| assert c4 == 10 | |
| assert more4 is False | |
| # ─── 5. MULTI-TENANT BOLA ISOLATION ─────────────────────────────────────────── | |
| def test_multi_tenant_events_access_isolation(): | |
| """Verifies that unauthorized tenants are blocked from retrieving other users' execution telemetry.""" | |
| run_id = f"run_tenant_isolated_{int(time.time()*1000)}" | |
| rev = "c" * 64 | |
| # Record event owned by user_alice | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_RUN_STARTED, | |
| message="Alice workflow started", | |
| owner_user_id="user_alice_uuid", | |
| ) | |
| meta = GLOBAL_EVENT_STORE.get_run_metadata(run_id) | |
| assert meta["owner_user_id"] == "user_alice_uuid" | |
| resp = client.get(f"/api/graph/events/{run_id}") | |
| assert resp.status_code == 200 | |
| data = resp.json() | |
| assert data["run_id"] == run_id | |
| assert len(data["events"]) >= 1 | |
| # ─── 6. PRE-FLIGHT VALIDATION REJECTION EVENTS ──────────────────────────────── | |
| def test_preflight_validation_rejection_events(): | |
| """Ensures pre-flight validation rejections emit validation_error diagnostics and run_failed event.""" | |
| invalid_run_id = f"run_invalid_preflight_{int(time.time()*1000)}" | |
| invalid_graph = { | |
| "run_id": invalid_run_id, | |
| "nodes": [ | |
| { | |
| "id": "node_dup", | |
| "type": "input", | |
| "data": {"value": 10}, | |
| }, | |
| { | |
| "id": "node_dup", # Duplicate ID violates DAG schema | |
| "type": "input", | |
| "data": {"value": 20}, | |
| }, | |
| ], | |
| "edges": [], | |
| } | |
| result = execute_workflow_graph(invalid_graph) | |
| assert result["success"] is False | |
| assert result["status"] == "failed" | |
| events, _, _ = GLOBAL_EVENT_STORE.get_events(invalid_run_id, since_seq=0, limit=50) | |
| assert len(events) >= 2 | |
| types = [e.event_type for e in events] | |
| assert EVENT_VALIDATION_ERROR in types | |
| assert EVENT_RUN_FAILED in types | |
| # ─── 7. TIMEOUT EVENT EMISSION ──────────────────────────────────────────────── | |
| def test_timeout_event_emission_and_node_cascade(): | |
| """Verifies that timeouts emit timeout and run_failed events while marking downstream nodes skipped.""" | |
| timeout_run_id = f"run_timeout_{int(time.time()*1000)}" | |
| graph = { | |
| "run_id": timeout_run_id, | |
| "timeout_seconds": 0.05, | |
| "nodes": [ | |
| { | |
| "id": "n1", | |
| "type": "primitive.variable", | |
| "data": {"type": "primitive.variable", "value": 10, "label": "Node 1"}, | |
| }, | |
| { | |
| "id": "n2", | |
| "type": "primitive.variable", | |
| "data": {"type": "primitive.variable", "value": 20, "label": "Node 2"}, | |
| }, | |
| ], | |
| "edges": [ | |
| {"id": "e1", "source": "n1", "target": "n2", "sourceHandle": "val", "targetHandle": "in_val"}, | |
| ], | |
| } | |
| orig_handler = we._execute_variable_node | |
| def slow_variable_node(inputs, params): | |
| time.sleep(0.1) | |
| return orig_handler(inputs, params) | |
| try: | |
| we._execute_variable_node = slow_variable_node | |
| result = execute_workflow_graph(graph, timeout_seconds=0.05) | |
| assert result["success"] is False | |
| events, _, _ = GLOBAL_EVENT_STORE.get_events(timeout_run_id, since_seq=0, limit=50) | |
| types = [e.event_type for e in events] | |
| assert EVENT_TIMEOUT in types | |
| assert EVENT_RUN_FAILED in types | |
| finally: | |
| we._execute_variable_node = orig_handler | |
| # ─── 8. CANCELLATION TELEMETRY ──────────────────────────────────────────────── | |
| def test_cancellation_telemetry_events(): | |
| """Verifies that cancel_workflow_run emits node_cancelled and run_cancelled events.""" | |
| cancel_run_id = f"run_cancel_test_{int(time.time()*1000)}" | |
| rev = "d" * 64 | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=cancel_run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_RUN_STARTED, | |
| message="Workflow started", | |
| ) | |
| node_results = { | |
| "node_1": {"node_id": "node_1", "status": "running", "exec_state": "running"}, | |
| "node_2": {"node_id": "node_2", "status": "queued", "exec_state": "queued"}, | |
| } | |
| res = cancel_workflow_run(run_id=cancel_run_id, node_results=node_results) | |
| assert res["status"] == "cancelled" | |
| assert "node_1" in res["cancelled_nodes"] | |
| assert "node_2" in res["cancelled_nodes"] | |
| events, _, _ = GLOBAL_EVENT_STORE.get_events(cancel_run_id, since_seq=0, limit=50) | |
| types = [e.event_type for e in events] | |
| assert EVENT_NODE_CANCELLED in types | |
| assert EVENT_RUN_CANCELLED in types | |
| # ─── 9. OUTPUT QUOTA EXCEEDED TELEMETRY ─────────────────────────────────────── | |
| def test_output_quota_exceeded_telemetry_event(): | |
| """Verifies that output payload exceeding 1MB emits quota_exceeded event with code ERR-OUTPUT-SIZE-EXCEEDED.""" | |
| oversized_run_id = f"run_quota_exceeded_{int(time.time()*1000)}" | |
| oversized_string = "X" * (1024 * 1024 + 100) # > 1MB | |
| graph = { | |
| "run_id": oversized_run_id, | |
| "nodes": [ | |
| { | |
| "id": "huge_node", | |
| "type": "constant.string", | |
| "data": {"label": "Oversized String", "parameters": {"literal_value": oversized_string, "const_type": "string"}}, | |
| }, | |
| ], | |
| "edges": [], | |
| } | |
| result = execute_workflow_graph(graph) | |
| assert result["success"] is False | |
| events, _, _ = GLOBAL_EVENT_STORE.get_events(oversized_run_id, since_seq=0, limit=50) | |
| types = [e.event_type for e in events] | |
| assert EVENT_QUOTA_EXCEEDED in types | |
| quota_events = [e for e in events if e.event_type == EVENT_QUOTA_EXCEEDED] | |
| assert len(quota_events) >= 1 | |
| assert quota_events[0].code == "ERR-OUTPUT-SIZE-EXCEEDED" | |
| # ─── 10. FASTAPI TELEMETRY ENDPOINTS INTEGRATION ────────────────────────────── | |
| def test_fastapi_runtime_feedback_endpoints(): | |
| """Tests the /graph/events/{run_id} and /graph/feedback/{run_id} endpoints.""" | |
| run_id = f"run_api_test_{int(time.time()*1000)}" | |
| rev = "e" * 64 | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_RUN_STARTED, | |
| message="API run started", | |
| ) | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_NODE_STARTED, | |
| node_id="n1", | |
| message="Executing node n1", | |
| ) | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_RUN_COMPLETED, | |
| message="API run completed", | |
| ) | |
| # 1. Test /graph/events | |
| ev_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=10") | |
| assert ev_resp.status_code == 200 | |
| ev_data = ev_resp.json() | |
| assert ev_data["run_id"] == run_id | |
| assert ev_data["graph_revision"] == rev | |
| assert len(ev_data["events"]) == 3 | |
| assert ev_data["cursor"] == 3 | |
| assert ev_data["is_complete"] is True | |
| # 2. Test /graph/feedback snapshot | |
| fb_resp = client.get(f"/api/graph/feedback/{run_id}") | |
| assert fb_resp.status_code == 200 | |
| fb_data = fb_resp.json() | |
| assert fb_data["run_id"] == run_id | |
| assert fb_data["graph_revision"] == rev | |
| assert fb_data["lifecycle_status"] == "success" | |
| assert fb_data["is_complete"] is True | |
| # 3. Test invalid run_id identifier rejected | |
| malformed_resp = client.get("/api/graph/events/../../etc/passwd") | |
| assert malformed_resp.status_code in (400, 404) | |
| # ─── 11. EDGE CASE 1: EVENT RETENTION BUFFER RECOVERY & GAP SIGNALING ───────── | |
| def test_edge_case_ring_buffer_retention_and_gap_recovery(): | |
| """ | |
| Verifies ring buffer behavior when events exceed MAX_EVENTS_PER_RUN (1,000). | |
| When client cursor is older than the oldest retained event, event store returns | |
| events_truncated=True, oldest_retained_seq, and endpoint issues warning. | |
| """ | |
| run_id = f"run_retention_gap_{int(time.time()*1000)}" | |
| rev = "f" * 64 | |
| # Generate 1,050 events (exceeding 1,000-event ring buffer) | |
| for i in range(1050): | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_NODE_STARTED if i % 2 == 0 else EVENT_NODE_COMPLETED, | |
| node_id=f"node_{i}", | |
| message=f"Event {i}", | |
| ) | |
| # Client requests from cursor 0 (which was pruned by ring buffer) | |
| res = GLOBAL_EVENT_STORE.get_events(run_id, since_seq=0, limit=50) | |
| assert res.events_truncated is True | |
| assert res.oldest_retained_seq > 0 | |
| assert len(res[0]) > 0 | |
| # First returned event matches oldest_retained_seq | |
| assert res[0][0].seq == res.oldest_retained_seq | |
| # Test API endpoint response includes warning and retention metadata | |
| api_resp = client.get(f"/api/graph/events/{run_id}?cursor=0&limit=50") | |
| assert api_resp.status_code == 200 | |
| data = api_resp.json() | |
| assert data["events_truncated"] is True | |
| assert data["oldest_retained_seq"] == res.oldest_retained_seq | |
| assert "warning" in data | |
| assert "truncated" in data["warning"] | |
| # ─── 12. EDGE CASE 2: ADVANCED SECRETS & NESTED REDACTION ──────────────────── | |
| def test_edge_case_advanced_credentials_and_nested_redaction(): | |
| """ | |
| Verifies sanitization of: | |
| - URL-encoded passwords in connection strings (e.g. mysql://user:p%40ssw0rd@host:3306) | |
| - HTTP Basic Auth headers (Basic dXNlcjpzZWNyZXQ=) | |
| - Generic multiline secret assignments (client_secret, webhook_secret, password) | |
| - Deeply nested structures (dicts inside lists inside dicts) | |
| """ | |
| from language_core.runtime_events import scrub_sensitive_string, sanitize_event_payload | |
| # 1. URL-encoded passwords in connection URIs | |
| uri_text = "Connecting to postgresql://db_user:MyP%40ssw0rd%21@db.internal.net:5432/production_db" | |
| scrubbed_uri = scrub_sensitive_string(uri_text) | |
| assert "MyP%40ssw0rd%21" not in scrubbed_uri | |
| assert "postgresql://db_user:[REDACTED_PASSWORD]@db.internal.net:5432/production_db" in scrubbed_uri | |
| # 2. HTTP Basic Auth header | |
| basic_header = "Authorization: Basic dXNlcjpwYXNzd29yZDEyMw==" | |
| scrubbed_basic = scrub_sensitive_string(basic_header) | |
| assert "Basic [REDACTED_BASIC_AUTH]" in scrubbed_basic | |
| assert "dXNlcjpwYXNzd29yZDEyMw==" not in scrubbed_basic | |
| # 3. Multiline secret assignments | |
| multiline_code = ( | |
| 'def setup_keys():\n' | |
| ' client_secret = "cs_live_998877665544332211"\n' | |
| ' webhook_secret = "whsec_abcdef1234567890"\n' | |
| ' password = "SuperSecretPassword!"\n' | |
| ' return True' | |
| ) | |
| scrubbed_multi = scrub_sensitive_string(multiline_code) | |
| assert "cs_live_998877665544332211" not in scrubbed_multi | |
| assert "whsec_abcdef1234567890" not in scrubbed_multi | |
| assert "SuperSecretPassword!" not in scrubbed_multi | |
| assert "[REDACTED]" in scrubbed_multi | |
| # 4. Deeply nested payloads | |
| nested_payload = { | |
| "config": { | |
| "services": [ | |
| { | |
| "name": "payment_gateway", | |
| "dsn": "postgres://pay_admin:TopSecretPass123@sql.prod:5432/payments", | |
| "tokens": {"api_key": "ak_live_xyz_9876543210"}, | |
| } | |
| ] | |
| } | |
| } | |
| sanitized = sanitize_event_payload(nested_payload) | |
| dsn_clean = sanitized["config"]["services"][0]["dsn"] | |
| assert "TopSecretPass123" not in dsn_clean | |
| assert "[REDACTED_PASSWORD]" in dsn_clean | |
| assert sanitized["config"]["services"][0]["tokens"]["api_key"] == "[REDACTED]" | |
| # ─── 13. EDGE CASE 3: MULTI-TENANT AUTHORIZATION & BOLA/IDOR DEFENSE ───────── | |
| def test_edge_case_multi_tenant_access_isolation(): | |
| """ | |
| Verifies that Tenant B is strictly forbidden (HTTP 403 ERR-FORBIDDEN-SESSION) | |
| from accessing Tenant A's: | |
| 1. /graph/events/{run_id} | |
| 2. /graph/feedback/{run_id} | |
| 3. /graph/events/{run_id}/stream | |
| While Tenant A and Admin can access them. | |
| """ | |
| run_id = f"run_bola_tenant_a_{int(time.time()*1000)}" | |
| tenant_a_id = "tenant_alice_uuid_111" | |
| tenant_b_id = "tenant_bob_uuid_222" | |
| rev = "a" * 64 | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_RUN_STARTED, | |
| message="Tenant A execution started", | |
| owner_user_id=tenant_a_id, | |
| ) | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_RUN_COMPLETED, | |
| message="Tenant A execution completed", | |
| owner_user_id=tenant_a_id, | |
| ) | |
| # 1. Tenant B attempts /graph/events -> 403 Forbidden | |
| res_b_events = client.get( | |
| f"/api/graph/events/{run_id}", | |
| headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"} | |
| ) | |
| assert res_b_events.status_code == 403 | |
| assert res_b_events.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION" | |
| # 2. Tenant B attempts /graph/feedback -> 403 Forbidden | |
| res_b_fb = client.get( | |
| f"/api/graph/feedback/{run_id}", | |
| headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"} | |
| ) | |
| assert res_b_fb.status_code == 403 | |
| assert res_b_fb.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION" | |
| # 3. Tenant B attempts /graph/events/{run_id}/stream -> 403 Forbidden | |
| res_b_stream = client.get( | |
| f"/api/graph/events/{run_id}/stream", | |
| headers={"x-user-id": tenant_b_id, "x-user-role": "authenticated"} | |
| ) | |
| assert res_b_stream.status_code == 403 | |
| assert res_b_stream.json()["detail"]["code"] == "ERR-FORBIDDEN-SESSION" | |
| # 4. Tenant A accesses their own run -> 200 OK | |
| res_a = client.get( | |
| f"/api/graph/events/{run_id}", | |
| headers={"x-user-id": tenant_a_id, "x-user-role": "authenticated"} | |
| ) | |
| assert res_a.status_code == 200 | |
| assert len(res_a.json()["events"]) >= 2 | |
| # 5. Admin accesses Tenant A's run -> 200 OK | |
| res_admin = client.get( | |
| f"/api/graph/events/{run_id}", | |
| headers={"x-user-id": "service_account", "x-user-role": "admin"} | |
| ) | |
| assert res_admin.status_code == 200 | |
| # ─── 14. EDGE CASE 4: OUTPUT QUOTA CASCADE ISOLATION ───────────────────────── | |
| def test_edge_case_output_quota_cascade_prevents_partial_propagation(): | |
| """ | |
| Verifies that when an upstream node fails the 1 MB output quota (ERR-OUTPUT-SIZE-EXCEEDED), | |
| the downstream node is SKIPPED with WARN-DEPENDENCY-FAILED and NEVER receives | |
| a truncated preview as an actual execution input. | |
| """ | |
| from language_core.workflow_executor import execute_workflow_graph | |
| run_id = f"run_quota_cascade_{int(time.time()*1000)}" | |
| # Node 1 produces oversized string (>1MB quota) | |
| # Node 2 depends on Node 1's output | |
| graph = { | |
| "run_id": run_id, | |
| "nodes": [ | |
| { | |
| "id": "node_oversized", | |
| "type": "input", | |
| "data": {"value": "X" * 1_050_000}, | |
| }, | |
| { | |
| "id": "node_downstream", | |
| "type": "output", | |
| "data": {}, | |
| }, | |
| ], | |
| "edges": [ | |
| { | |
| "id": "e_oversized_downstream", | |
| "source": "node_oversized", | |
| "target": "node_downstream", | |
| "sourceHandle": "output", | |
| "targetHandle": "input", | |
| } | |
| ], | |
| } | |
| results = execute_workflow_graph(graph, timeout_seconds=5.0) | |
| assert results["success"] is False | |
| assert results["status"] == "failed" | |
| # Node 1 must be failed with ERR-OUTPUT-SIZE-EXCEEDED | |
| oversized_res = results["node_results"]["node_oversized"] | |
| assert oversized_res["status"] == "failed" | |
| assert "ERR-OUTPUT-SIZE-EXCEEDED" in oversized_res["error"] | |
| assert oversized_res.get("error_semantics", {}).get("code") == "ERR-OUTPUT-SIZE-EXCEEDED" | |
| # Node 2 must be skipped with WARN-DEPENDENCY-FAILED and NO inputs consumed | |
| downstream_res = results["node_results"]["node_downstream"] | |
| assert downstream_res["status"] == "skipped" | |
| assert "Upstream dependency" in downstream_res["error"] | |
| assert downstream_res.get("error_semantics", {}).get("code") == "WARN-DEPENDENCY-FAILED" | |
| assert downstream_res["inputs"] == {} | |
| # ─── 15. EDGE CASE 5: SSE STREAM RECONNECTION & CURSOR RECOVERY ────────────── | |
| def test_edge_case_sse_streaming_reconnection_and_terminal_packet(): | |
| """ | |
| Verifies SSE stream reconnection behavior: | |
| 1. Resumption via Last-Event-ID header | |
| 2. Resumption via cursor query parameter | |
| 3. Proper SSE event packaging (id:, event:, data:) | |
| 4. Terminal stream_end event emission upon completion | |
| """ | |
| run_id = f"run_sse_recon_{int(time.time()*1000)}" | |
| rev = "b" * 64 | |
| # Record 5 events | |
| for i in range(1, 6): | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, | |
| graph_revision=rev, | |
| event_type=EVENT_NODE_STARTED if i < 5 else EVENT_RUN_COMPLETED, | |
| node_id=f"node_{i}", | |
| message=f"Step {i}", | |
| ) | |
| # Reconnect using Last-Event-ID = 3 (expect events 4 and 5, then stream_end) | |
| with client.stream("GET", f"/api/graph/events/{run_id}/stream", headers={"last-event-id": "3"}) as response: | |
| assert response.status_code == 200 | |
| assert "text/event-stream" in response.headers["content-type"] | |
| lines = [] | |
| for line in response.iter_lines(): | |
| if line: | |
| lines.append(line) | |
| if "stream_end" in line: | |
| break | |
| stream_output = "\n".join(lines) | |
| # Check that events 4 and 5 are received with valid id tags | |
| assert "id: 4" in stream_output or "node_4" in stream_output | |
| assert "id: 5" in stream_output or "node_5" in stream_output | |
| # Check terminal packet | |
| assert "event: stream_end" in stream_output | |