File size: 8,762 Bytes
8628c98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | """
ALLI Workflow Executor: Compiled Single-Path Graph Execution Engine.
Synthesizes standalone Python via codegen_handler and executes via RuntimeRunner
for full parity across local_subprocess and docker_sandbox modes.
"""
import time
from typing import Any, Dict, List, Optional, Set
from language_core.runtime_events import (
EVENT_RUN_COMPLETED,
EVENT_RUN_FAILED,
GLOBAL_EVENT_STORE,
)
from runtime_environment.active_runs import GLOBAL_ACTIVE_RUNS
from runtime_environment.runner import DockerSandboxRunner, RuntimeRunner
def _slice_upstream_subgraph(graph: Dict[str, Any], target_node_id: str) -> Dict[str, Any]:
"""Slice graph to include target_node_id and all upstream dependency nodes/edges."""
raw_nodes = graph.get("nodes") or []
raw_edges = graph.get("edges") if graph.get("edges") is not None else (graph.get("connections") or [])
upstream: Set[str] = {str(target_node_id)}
changed = True
while changed:
changed = False
for e in raw_edges:
src = str(e.get("source") or e.get("from_node") or "")
tgt = str(e.get("target") or e.get("to_node") or "")
if tgt in upstream and src and src not in upstream:
upstream.add(src)
changed = True
return {
**graph,
"nodes": [n for n in raw_nodes if str(n.get("id")) in upstream],
"edges": [
e for e in raw_edges
if str(e.get("source") or e.get("from_node")) in upstream
and str(e.get("target") or e.get("to_node")) in upstream
],
}
def execute_compiled_workflow_graph(
graph: Dict[str, Any],
timeout_seconds: float,
mode: str,
run_id: str,
graph_revision: str,
created_ts: str,
start_total_time: float,
target_node_id: Optional[str] = None,
owner_user_id: Optional[str] = None,
workspace_env: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
"""Compile graph to Python and run via RuntimeRunner (local_subprocess or docker_sandbox)."""
from language_core.executor.run_prep import scrub_result_secrets
from lambdas.codegen_handler import generate_code_logic
norm_mode = mode.strip().lower()
if norm_mode in ("docker", "docker_sandbox"):
runner_mode = "docker"
mode_label = "docker_sandbox"
elif norm_mode in ("local_subprocess", "local_compiled"):
runner_mode = "local"
mode_label = "local_subprocess"
else:
runner_mode = "auto"
mode_label = "docker_sandbox" if DockerSandboxRunner.is_docker_available() else "local_subprocess"
exec_graph = _slice_upstream_subgraph(graph, target_node_id) if target_node_id else graph
codegen_res = generate_code_logic(graph=exec_graph, validate=True)
execution_order: List[str] = list(codegen_res.get("execution_order") or [])
if not codegen_res.get("success") or not codegen_res.get("python_code"):
duration_ms = round((time.perf_counter() - start_total_time) * 1000, 2)
diags = codegen_res.get("diagnostics") or []
err_msgs = [d.get("message", "Codegen/validation error") for d in diags if d.get("severity") == "error"]
err_msg = "; ".join(err_msgs[:3]) or "Failed to synthesize executable Python from workflow graph."
GLOBAL_EVENT_STORE.record_event(
run_id=run_id, graph_revision=graph_revision, event_type=EVENT_RUN_FAILED,
status="failed", code="CODEGEN-FAILED", message=err_msg, payload={"diagnostics": diags},
owner_user_id=owner_user_id,
)
return {
"run_id": run_id,
"graph_revision": graph_revision,
"cursor": GLOBAL_EVENT_STORE._seq_counters.get(run_id, 0),
"success": False,
"status": "failed",
"mode": mode_label,
"duration_ms": duration_ms,
"total_duration_ms": duration_ms,
"execution_order": execution_order,
"execution_plan": None,
"node_results": {},
"validation_errors": diags,
"diagnostics": diags,
"final_outputs": {},
"logs": [f"[ERROR] {err_msg}"],
"stdout": "",
"stderr": err_msg,
"exit_code": 1,
"error": err_msg,
"python_code": "",
}
python_code = codegen_res["python_code"]
runner = RuntimeRunner(default_mode=runner_mode, timeout_sec=timeout_seconds, run_id=run_id, workspace_env=workspace_env)
GLOBAL_ACTIVE_RUNS.register(run_id)
try:
exec_res = runner.run(python_code, mode=runner_mode)
except RuntimeError as exc:
duration_ms = round((time.perf_counter() - start_total_time) * 1000, 2)
err_str = str(exc)
err_code = "ERR-SANDBOX-UNAVAILABLE" if "ERR-SANDBOX-UNAVAILABLE" in err_str else "ERR-SANDBOX-FAILED"
GLOBAL_EVENT_STORE.record_event(
run_id=run_id, graph_revision=graph_revision, event_type=EVENT_RUN_FAILED,
status="failed", severity="error", code=err_code, message=err_str,
owner_user_id=owner_user_id,
)
return {
"run_id": run_id,
"graph_revision": graph_revision,
"cursor": GLOBAL_EVENT_STORE._seq_counters.get(run_id, 0),
"success": False,
"status": "failed",
"mode": mode_label,
"code": err_code,
"duration_ms": duration_ms,
"total_duration_ms": duration_ms,
"execution_order": execution_order,
"execution_plan": None,
"node_results": {},
"final_outputs": {},
"logs": [f"[ERROR] {err_str}"],
"stdout": "",
"stderr": err_str,
"exit_code": 1,
"error": err_str,
"python_code": python_code,
}
finally:
GLOBAL_ACTIVE_RUNS.unregister(run_id)
total_ms = round((time.perf_counter() - start_total_time) * 1000, 2)
per_node_ms = round(exec_res.duration_ms / max(len(execution_order), 1), 2)
node_map = {str(n.get("id")): n for n in (exec_graph.get("nodes") or []) if n.get("id")}
node_results: Dict[str, Any] = {}
status_str = "success" if exec_res.success else "failed"
for nid in execution_order:
n = node_map.get(nid, {})
d = n.get("data") or {}
ntype = str(n.get("node_type") or n.get("type") or d.get("node_type") or "").lower()
lbl = d.get("label") or n.get("label") or ntype or nid
outs = {"stdout": exec_res.stdout} if (exec_res.success and ("print" in ntype or "output" in ntype)) else {}
node_results[nid] = {
"run_id": run_id,
"node_id": nid,
"node_type": ntype,
"label": lbl,
"status": status_str,
"config_state": "valid",
"exec_state": status_str,
"freshness": "current" if exec_res.success else "invalidated",
"inputs": {},
"outputs": outs,
"output_ports": {},
"duration_ms": per_node_ms,
"retries_attempted": 0,
"error": None if exec_res.success else (exec_res.stderr or "Execution failed"),
"error_semantics": None,
"skip_reason": None,
"timestamp": created_ts,
}
GLOBAL_EVENT_STORE.record_event(
run_id=run_id,
graph_revision=graph_revision,
event_type=EVENT_RUN_COMPLETED if exec_res.success else EVENT_RUN_FAILED,
status=status_str,
message=f"Compiled workflow ({exec_res.mode}) exited {exec_res.exit_code} in {round(exec_res.duration_ms, 2)}ms.",
payload={"mode": exec_res.mode, "exit_code": exec_res.exit_code, "duration_ms": round(exec_res.duration_ms, 2)},
owner_user_id=owner_user_id,
)
return scrub_result_secrets({
"run_id": run_id,
"graph_revision": graph_revision,
"cursor": GLOBAL_EVENT_STORE._seq_counters.get(run_id, 0),
"success": exec_res.success,
"status": status_str,
"mode": exec_res.mode,
"duration_ms": round(exec_res.duration_ms, 2),
"total_duration_ms": total_ms,
"execution_order": execution_order,
"execution_plan": None,
"node_results": node_results,
"final_outputs": {"stdout": exec_res.stdout} if exec_res.success else {},
"logs": [f"[{exec_res.mode}] Exit {exec_res.exit_code} in {round(exec_res.duration_ms, 2)}ms"],
"stdout": exec_res.stdout[:100000],
"stderr": exec_res.stderr[:100000],
"exit_code": exec_res.exit_code,
"error": None if exec_res.success else (exec_res.stderr or "Compiled workflow execution failed."),
"python_code": python_code,
})
|