rsnarsna commited on
Commit
c9366ea
·
1 Parent(s): 355ed30

test(phase5): add adversarial regression suite for step 5.5 cycle & preflight invariants

Browse files
Files changed (1) hide show
  1. tests/test_adversarial_step5_5.py +398 -0
tests/test_adversarial_step5_5.py ADDED
@@ -0,0 +1,398 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ALLI Phase 5 Step 5.5 — Adversarial Review & Invariant Verification Suite
3
+ Tests all 9 adversarial invariants against the live backend implementation:
4
+ 1. Pure procedural cycles & disconnected cycles -> DAG-001, can_execute: False
5
+ 2. Approved agent reasoning cycles -> Permitted within approved structures
6
+ 3. Hybrid agent/procedural cycles -> Rejected with DAG-001
7
+ 4. Invalid loop-header feedback -> Consistently produces CTRL-001
8
+ 5. Fail-closed execution preflight contracts
9
+ 6. Direct graph execution & step execution preflight enforcement
10
+ 7. Planner & executor preflight rejection before starting subprocesses or side effects
11
+ 8. Diagnostic codes and execution eligibility consistency
12
+ 9. Legitimate procedural, loop, and agent workflows remain fully functional
13
+ """
14
+
15
+ import pytest
16
+ from lambdas.validate_handler import validate_workflow_protocol_logic, validate_connection_logic
17
+ from language_core.topology import TopologyGraph, Node, InvalidConnectionError, CycleDetectedError
18
+ from language_core.types import TYPE_ANY
19
+ from language_core.execution_planner import build_execution_plan
20
+ from language_core.workflow_executor import execute_workflow_graph, execute_workflow_step
21
+
22
+
23
+ # ==============================================================================
24
+ # INVARIANT 1: Pure Procedural Cycles & Disconnected Cycles Rejected with DAG-001
25
+ # ==============================================================================
26
+
27
+ def test_inv1_pure_procedural_cycles_rejected_with_dag001():
28
+ # 2-node cycle
29
+ g_2node = {
30
+ "nodes": [
31
+ {"id": "a", "node_type": "math.add", "data": {"label": "Add A"}},
32
+ {"id": "b", "node_type": "math.add", "data": {"label": "Add B"}},
33
+ ],
34
+ "edges": [
35
+ {"id": "e1", "source": "a", "target": "b"},
36
+ {"id": "e2", "source": "b", "target": "a"},
37
+ ],
38
+ }
39
+ res2 = validate_workflow_protocol_logic(g_2node)
40
+ assert not res2["valid"]
41
+ assert not res2["can_execute"]
42
+ assert any(d["code"] == "DAG-001" for d in res2["diagnostics"])
43
+
44
+ # 4-node long cycle
45
+ g_4node = {
46
+ "nodes": [
47
+ {"id": "n1", "node_type": "math.add", "data": {"label": "Node 1"}},
48
+ {"id": "n2", "node_type": "math.multiply", "data": {"label": "Node 2"}},
49
+ {"id": "n3", "node_type": "math.subtract", "data": {"label": "Node 3"}},
50
+ {"id": "n4", "node_type": "math.divide", "data": {"label": "Node 4"}},
51
+ ],
52
+ "edges": [
53
+ {"id": "e1", "source": "n1", "target": "n2"},
54
+ {"id": "e2", "source": "n2", "target": "n3"},
55
+ {"id": "e3", "source": "n3", "target": "n4"},
56
+ {"id": "e4", "source": "n4", "target": "n1"},
57
+ ],
58
+ }
59
+ res4 = validate_workflow_protocol_logic(g_4node)
60
+ assert not res4["valid"]
61
+ assert not res4["can_execute"]
62
+ dag_diags = [d for d in res4["diagnostics"] if d["code"] == "DAG-001"]
63
+ assert len(dag_diags) > 0
64
+ assert "Node 1" in dag_diags[0]["message"]
65
+
66
+
67
+ def test_inv1_disconnected_procedural_cycles_rejected():
68
+ # Linear component (v1 -> p1) + Disconnected cycle (c1 <-> c2)
69
+ g_disc = {
70
+ "nodes": [
71
+ {"id": "v1", "node_type": "constant.int", "data": {"label": "Var", "customData": {"var_name": "x", "value": 10}}},
72
+ {"id": "p1", "node_type": "core.print", "data": {"label": "Print"}},
73
+ {"id": "c1", "node_type": "math.multiply", "data": {"label": "Cycle 1"}},
74
+ {"id": "c2", "node_type": "math.multiply", "data": {"label": "Cycle 2"}},
75
+ ],
76
+ "edges": [
77
+ {"id": "e_valid", "source": "v1", "source_handle": "val", "target": "p1", "target_handle": "value"},
78
+ {"id": "e_c1", "source": "c1", "target": "c2"},
79
+ {"id": "e_c2", "source": "c2", "target": "c1"},
80
+ ],
81
+ }
82
+ res = validate_workflow_protocol_logic(g_disc)
83
+ assert not res["valid"]
84
+ assert not res["can_execute"]
85
+ assert any(d["code"] == "DAG-001" for d in res["diagnostics"])
86
+
87
+
88
+ def test_inv1_self_loop_procedural_node():
89
+ # Direct self connection on ordinary node
90
+ g_self = {
91
+ "nodes": [
92
+ {"id": "m1", "node_type": "math.add", "data": {"label": "Add Self"}},
93
+ ],
94
+ "edges": [
95
+ {"id": "e_self", "source": "m1", "target": "m1"},
96
+ ],
97
+ }
98
+ nodes_map = {"m1": g_self["nodes"][0]}
99
+ conn_res = validate_connection_logic(
100
+ connection={"source": "m1", "target": "m1", "from_node": "m1", "to_node": "m1"},
101
+ nodes_map=nodes_map,
102
+ )
103
+ assert not conn_res["valid"]
104
+ assert conn_res["code"] == "DAG-001"
105
+
106
+
107
+ # ==============================================================================
108
+ # INVARIANT 2: Approved Agent Reasoning Cycles Permitted ONLY Within Approved Structures
109
+ # ==============================================================================
110
+
111
+ def test_inv2_approved_agent_reasoning_cycle_permitted():
112
+ agent_graph = {
113
+ "name": "Approved Agent Graph",
114
+ "nodes": [
115
+ {"id": "prompt_in", "node_type": "io.input", "data": {"label": "Prompt", "customData": {"value": "What is 2+2?"}}},
116
+ {"id": "agent_llm", "node_type": "alli.agent.llm", "data": {"label": "LLM", "category": "Agent Logic", "customData": {"model": "gpt-4o"}}},
117
+ {
118
+ "id": "agent_tool",
119
+ "node_type": "alli.agent.tool",
120
+ "data": {
121
+ "label": "Calculator",
122
+ "category": "Agent Tools",
123
+ "customData": {"tool_name": "calc"},
124
+ "spec": {
125
+ "inputs": [{"id": "query", "name": "query", "type": "str", "required": False}],
126
+ "outputs": [{"id": "result", "name": "result", "type": "str"}],
127
+ },
128
+ },
129
+ },
130
+ ],
131
+ "edges": [
132
+ {"id": "e_in", "source": "prompt_in", "source_handle": "output", "target": "agent_llm", "target_handle": "prompt"},
133
+ {"id": "e_fwd", "source": "agent_llm", "source_handle": "response", "target": "agent_tool", "target_handle": "query"},
134
+ {"id": "e_back", "source": "agent_tool", "source_handle": "result", "target": "agent_llm", "target_handle": "messages"},
135
+ ],
136
+ }
137
+ res = validate_workflow_protocol_logic(agent_graph)
138
+ assert not any(d["code"] in ("DAG-001", "CTRL-001") for d in res["diagnostics"])
139
+
140
+ plan = build_execution_plan(agent_graph)
141
+ assert plan["valid"] is True
142
+ assert plan["can_execute"] is True
143
+ assert plan["execution_order"] == ["prompt_in", "agent_llm", "agent_tool"]
144
+
145
+
146
+ def test_inv2_tool_to_tool_cycle_without_reasoner_rejected():
147
+ # Two tools cycling with each other without any LLM, reasoner, or memory
148
+ two_tools_graph = {
149
+ "name": "Two Tools Cycle",
150
+ "nodes": [
151
+ {"id": "prompt_in", "node_type": "io.input", "data": {"label": "Prompt", "customData": {"value": "test"}}},
152
+ {
153
+ "id": "tool_1",
154
+ "node_type": "alli.agent.tool",
155
+ "data": {"label": "Tool 1", "category": "Agent Tools", "spec": {"inputs": [{"id": "in", "name": "in", "required": False}]}},
156
+ },
157
+ {
158
+ "id": "tool_2",
159
+ "node_type": "alli.agent.tool",
160
+ "data": {"label": "Tool 2", "category": "Agent Tools", "spec": {"inputs": [{"id": "in", "name": "in", "required": False}]}},
161
+ },
162
+ ],
163
+ "edges": [
164
+ {"id": "e0", "source": "prompt_in", "target": "tool_1"},
165
+ {"id": "e1", "source": "tool_1", "target": "tool_2"},
166
+ {"id": "e2", "source": "tool_2", "target": "tool_1"},
167
+ ],
168
+ }
169
+ res = validate_workflow_protocol_logic(two_tools_graph)
170
+ assert not res["valid"]
171
+ assert not res["can_execute"]
172
+ assert any(d["code"] == "DAG-001" for d in res["diagnostics"])
173
+
174
+
175
+ # ==============================================================================
176
+ # INVARIANT 3: Hybrid Agent/Procedural Cycles Rejected
177
+ # ==============================================================================
178
+
179
+ def test_inv3_hybrid_agent_procedural_cycle_rejected():
180
+ # LLM Reasoner -> math.add -> LLM Reasoner
181
+ hybrid_graph = {
182
+ "name": "Hybrid Cycle",
183
+ "nodes": [
184
+ {"id": "agent_llm", "node_type": "alli.agent.llm", "data": {"label": "LLM Reasoner", "category": "Agent Logic"}},
185
+ {"id": "math_add", "node_type": "math.add", "data": {"label": "Add Node", "category": "Math"}},
186
+ ],
187
+ "edges": [
188
+ {"id": "e1", "source": "agent_llm", "target": "math_add"},
189
+ {"id": "e2", "source": "math_add", "target": "agent_llm"},
190
+ ],
191
+ }
192
+ res = validate_workflow_protocol_logic(hybrid_graph)
193
+ assert not res["valid"]
194
+ assert not res["can_execute"]
195
+ dag_diags = [d for d in res["diagnostics"] if d["code"] == "DAG-001"]
196
+ assert len(dag_diags) > 0
197
+ assert "Dependency Cycle Detected" in dag_diags[0]["message"]
198
+
199
+
200
+ def test_inv3_multi_node_hybrid_cycle_rejected():
201
+ # LLM -> Tool -> Math -> LLM
202
+ multi_hybrid = {
203
+ "nodes": [
204
+ {"id": "llm", "node_type": "alli.agent.llm", "data": {"label": "LLM", "category": "Agent Logic"}},
205
+ {"id": "tool", "node_type": "alli.agent.tool", "data": {"label": "Tool", "category": "Agent Tools"}},
206
+ {"id": "proc", "node_type": "core.transform", "data": {"label": "Procedural Transform"}},
207
+ ],
208
+ "edges": [
209
+ {"id": "e1", "source": "llm", "target": "tool"},
210
+ {"id": "e2", "source": "tool", "target": "proc"},
211
+ {"id": "e3", "source": "proc", "target": "llm"},
212
+ ],
213
+ }
214
+ res = validate_workflow_protocol_logic(multi_hybrid)
215
+ assert not res["valid"]
216
+ assert not res["can_execute"]
217
+ assert any(d["code"] == "DAG-001" for d in res["diagnostics"])
218
+
219
+
220
+ # ==============================================================================
221
+ # INVARIANT 4: Invalid Loop-Header Feedback Consistently Produces CTRL-001
222
+ # ==============================================================================
223
+
224
+ def test_inv4_for_loop_header_feedback_ctrl001():
225
+ # For loop header feedback wire
226
+ for_graph = {
227
+ "nodes": [
228
+ {"id": "for_hdr", "node_type": "control.for_loop", "data": {"label": "ForLoop Header", "customData": {"range_count": 5}}},
229
+ {"id": "body_step", "node_type": "core.print", "data": {"label": "Print Body"}},
230
+ ],
231
+ "edges": [
232
+ {"id": "e_fwd", "source": "for_hdr", "source_handle": "item", "target": "body_step", "target_handle": "value"},
233
+ {"id": "e_back", "source": "body_step", "source_handle": "out", "target": "for_hdr", "target_handle": "iterable"},
234
+ ],
235
+ }
236
+ res = validate_workflow_protocol_logic(for_graph)
237
+ assert not res["valid"]
238
+ assert not res["can_execute"]
239
+ ctrl_diags = [d for d in res["diagnostics"] if d["code"] == "CTRL-001"]
240
+ assert len(ctrl_diags) > 0
241
+ assert ctrl_diags[0]["node_id"] == "for_hdr"
242
+ assert ctrl_diags[0]["port_id"] == "iterable"
243
+ assert ctrl_diags[0]["edge_id"] == "e_back"
244
+
245
+
246
+ def test_inv4_while_loop_header_feedback_ctrl001():
247
+ # While loop header feedback wire
248
+ while_graph = {
249
+ "nodes": [
250
+ {"id": "while_hdr", "node_type": "control.while_loop", "data": {"label": "While Header", "customData": {"condition": True}}},
251
+ {"id": "step_1", "node_type": "math.add", "data": {"label": "Step 1"}},
252
+ ],
253
+ "edges": [
254
+ {"id": "e1", "source": "while_hdr", "target": "step_1"},
255
+ {"id": "e2", "source": "step_1", "target": "while_hdr"},
256
+ ],
257
+ }
258
+ res = validate_workflow_protocol_logic(while_graph)
259
+ assert not res["valid"]
260
+ assert not res["can_execute"]
261
+ ctrl_diags = [d for d in res["diagnostics"] if d["code"] == "CTRL-001"]
262
+ assert len(ctrl_diags) > 0
263
+ assert ctrl_diags[0]["node_id"] == "while_hdr"
264
+
265
+
266
+ def test_inv4_topology_graph_raises_ctrl001_on_loop():
267
+ top = TopologyGraph()
268
+ n_for = Node("for_node", label="For Loop", node_type="control.for_loop")
269
+ n_for.add_input("iterable", TYPE_ANY)
270
+ n_for.add_output("item", TYPE_ANY)
271
+
272
+ n_step = Node("step_node", label="Print Node", node_type="core.print")
273
+ n_step.add_input("value", TYPE_ANY)
274
+ n_step.add_output("out", TYPE_ANY)
275
+
276
+ top.add_node(n_for)
277
+ top.add_node(n_step)
278
+ top.connect("for_node", "item", "step_node", "value")
279
+
280
+ # Connecting back to loop header must raise InvalidConnectionError with code CTRL-001
281
+ with pytest.raises(InvalidConnectionError) as exc_info:
282
+ top.connect("step_node", "out", "for_node", "iterable")
283
+ assert exc_info.value.code == "CTRL-001"
284
+
285
+
286
+ # ==============================================================================
287
+ # INVARIANT 6: Direct Backend Graph Execution & Step Execution Enforce Preflight
288
+ # ==============================================================================
289
+
290
+ def test_inv6_direct_backend_graph_execution_enforces_preflight():
291
+ invalid_graph = {
292
+ "nodes": [
293
+ {"id": "x", "node_type": "math.add", "data": {"label": "X"}},
294
+ {"id": "y", "node_type": "math.add", "data": {"label": "Y"}},
295
+ ],
296
+ "edges": [
297
+ {"id": "e1", "source": "x", "target": "y"},
298
+ {"id": "e2", "source": "y", "target": "x"},
299
+ ],
300
+ }
301
+ res = execute_workflow_graph(invalid_graph)
302
+ assert res["success"] is False
303
+ assert res["status"] == "failed"
304
+ assert res["exit_code"] == 1
305
+ assert "Workflow validation failed" in res["error"]
306
+ assert len(res["final_outputs"]) == 0
307
+
308
+
309
+ def test_inv6_direct_backend_step_execution_enforces_preflight():
310
+ invalid_graph = {
311
+ "nodes": [
312
+ {"id": "x", "node_type": "math.add", "data": {"label": "X"}},
313
+ {"id": "y", "node_type": "math.add", "data": {"label": "Y"}},
314
+ ],
315
+ "edges": [
316
+ {"id": "e1", "source": "x", "target": "y"},
317
+ {"id": "e2", "source": "y", "target": "x"},
318
+ ],
319
+ }
320
+ res = execute_workflow_step(invalid_graph, target_node_id="x")
321
+ assert res["success"] is False
322
+ assert res["status"] == "failed"
323
+ assert res["exit_code"] == 1
324
+ assert "Workflow step execution blocked" in res["logs"][0]
325
+
326
+
327
+ # ==============================================================================
328
+ # INVARIANT 7: Planner Rejects Invalid Graphs Before Subprocesses / Side Effects
329
+ # ==============================================================================
330
+
331
+ def test_inv7_planner_preflight_blocks_staging_and_execution():
332
+ cyclic_graph = {
333
+ "nodes": [
334
+ {"id": "a", "node_type": "math.multiply", "data": {"label": "A"}},
335
+ {"id": "b", "node_type": "math.multiply", "data": {"label": "B"}},
336
+ ],
337
+ "edges": [
338
+ {"id": "e1", "source": "a", "target": "b"},
339
+ {"id": "e2", "source": "b", "target": "a"},
340
+ ],
341
+ }
342
+ plan = build_execution_plan(cyclic_graph)
343
+ assert plan["valid"] is False
344
+ assert plan["can_execute"] is False
345
+ assert plan["execution_order"] == []
346
+ assert plan["stages"] == []
347
+ assert len(plan["validation_errors"]) > 0
348
+
349
+
350
+ # ==============================================================================
351
+ # INVARIANT 8: Diagnostic Codes Consistency
352
+ # ==============================================================================
353
+
354
+ def test_inv8_diagnostic_codes_and_fields_consistent():
355
+ graph_cycle = {
356
+ "nodes": [
357
+ {"id": "node_a", "node_type": "math.add", "data": {"label": "Add A"}},
358
+ {"id": "node_b", "node_type": "math.add", "data": {"label": "Add B"}},
359
+ ],
360
+ "edges": [
361
+ {"id": "e1", "source": "node_a", "target": "node_b"},
362
+ {"id": "e2", "source": "node_b", "target": "node_a"},
363
+ ],
364
+ }
365
+ res = validate_workflow_protocol_logic(graph_cycle)
366
+ diag = next(d for d in res["diagnostics"] if d["code"] == "DAG-001")
367
+ assert diag["severity"] == "error"
368
+ assert diag["category"] == "Connection Validation"
369
+ assert diag["node_id"] in ("node_a", "node_b")
370
+ assert "fix_suggestion" in diag or "fix" in diag
371
+
372
+
373
+ # ==============================================================================
374
+ # INVARIANT 9: Legitimate Procedural, Loop, and Agent Workflows Remain Functional
375
+ # ==============================================================================
376
+
377
+ def test_inv9_legitimate_procedural_workflow_executes_successfully():
378
+ valid_proc = {
379
+ "name": "Valid Procedural Pipeline",
380
+ "nodes": [
381
+ {"id": "v1", "node_type": "constant.int", "data": {"label": "Const", "customData": {"var_name": "val", "value": 49}}},
382
+ {"id": "m1", "node_type": "math.sqrt", "data": {"label": "Sqrt"}},
383
+ {"id": "p1", "node_type": "core.print", "data": {"label": "Print"}},
384
+ ],
385
+ "edges": [
386
+ {"id": "e1", "source": "v1", "source_handle": "val", "target": "m1", "target_handle": "x"},
387
+ {"id": "e2", "source": "m1", "source_handle": "result", "target": "p1", "target_handle": "value"},
388
+ ],
389
+ }
390
+ val = validate_workflow_protocol_logic(valid_proc)
391
+ assert val["valid"] is True, f"Valid workflow rejected: {val['diagnostics']}"
392
+ assert val["can_execute"] is True
393
+
394
+ exec_res = execute_workflow_graph(valid_proc)
395
+ assert exec_res["success"] is True
396
+ assert exec_res["exit_code"] == 0
397
+ assert exec_res["node_results"]["m1"]["outputs"]["result"] == 7.0
398
+ assert "7.0" in exec_res["stdout"]