File size: 11,478 Bytes
8628c98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 | """
ALLI Workflow Executor Prep & Telemetry Hygiene Helper
Handles cached upstream/pinned Studio test state for isolated single-node `Test step`
runs and value-aware egress secret scrubbing across execution results.
"""
from collections import deque
from typing import Any, Dict, List, Optional, Set, Tuple
from language_core.execution_planner import build_execution_plan
from language_core.executor.state_and_errors import make_error_semantics
from language_core.runtime_events import (
EVENT_NODE_CANCELLED,
EVENT_NODE_QUEUED,
EVENT_RUN_CANCELLED,
GLOBAL_EVENT_STORE,
redact_sensitive_data,
redact_sensitive_text,
)
from runtime_environment.env_injection import (
RunEnvContext,
redact_secrets_from_obj,
redact_secrets_from_text,
)
_BLOCKED_PIN_KEYS = frozenset({"enabled", "source", "__proto__", "constructor", "prototype"})
def _extract_pinned_map(node: Dict[str, Any], field_name: str, *, require_explicit_enable: bool) -> Dict[str, Any]:
if not isinstance(node, dict):
return {}
data = node.get("data") if isinstance(node.get("data"), dict) else {}
raw_pin = data.get(field_name) if isinstance(data.get(field_name), dict) else node.get(field_name)
if not isinstance(raw_pin, dict):
return {}
if (require_explicit_enable and not raw_pin.get("enabled", True)) or ("enabled" in raw_pin and not raw_pin.get("enabled", True)):
return {}
inner = raw_pin.get("data") if isinstance(raw_pin.get("data"), dict) else raw_pin
return {str(k): v for k, v in inner.items() if str(k) not in _BLOCKED_PIN_KEYS}
def extract_node_pinned_input(node: Dict[str, Any]) -> Dict[str, Any]:
"""Read Studio test-only pinned input dict from node/data (never baked into codegen)."""
return _extract_pinned_map(node, "_pinned_input", require_explicit_enable=False)
def extract_node_pinned_output(node: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Read Studio test-only pinned output dict from node/data when explicitly enabled."""
clean = _extract_pinned_map(node, "_pinned_output", require_explicit_enable=True)
return clean if clean else None
def prepare_isolated_step_execution(
*,
execution_order: List[str],
node_map: Dict[str, Dict[str, Any]],
target_node_id: Optional[str],
single_node_only: bool,
cached_outputs: Optional[Dict[str, Dict[str, Any]]],
pinned_inputs: Optional[Dict[str, Dict[str, Any]]],
node_outputs_store: Dict[str, Dict[str, Any]],
) -> List[str]:
"""Seed upstream cached/pinned outputs ONLY for isolated single-node Studio `Test step` execution."""
if not single_node_only:
return execution_order
for nid, node in node_map.items():
if nid == target_node_id:
continue
pinned_out = extract_node_pinned_output(node)
if pinned_out:
node_outputs_store[nid] = dict(pinned_out)
if isinstance(cached_outputs, dict):
for nid, out_map in cached_outputs.items():
if str(nid) in node_map and isinstance(out_map, dict):
node_outputs_store[str(nid)] = {str(k): v for k, v in out_map.items() if str(k) not in _BLOCKED_PIN_KEYS}
if target_node_id and target_node_id in node_map:
if isinstance(pinned_inputs, dict) and isinstance(pinned_inputs.get(target_node_id), dict):
target_node = node_map[target_node_id]
data = target_node.setdefault("data", {})
if isinstance(data, dict):
clean_pins = {str(k): v for k, v in pinned_inputs[target_node_id].items() if str(k) not in _BLOCKED_PIN_KEYS}
data["_pinned_input"] = {"enabled": True, "data": clean_pins}
return [target_node_id]
return execution_order
def resolve_mapped_node_params(
params: Dict[str, Any],
data: Dict[str, Any],
node_outputs_store: Dict[str, Dict[str, Any]],
) -> Dict[str, Any]:
"""Resolve Stage-2 pill-mapped `_mapped_params` references against live/cached upstream outputs."""
mapped = (
params.get("_mapped_params")
or data.get("_mapped_params")
or (data.get("custom") if isinstance(data.get("custom"), dict) else {}).get("_mapped_params")
)
if not isinstance(mapped, dict) or not mapped:
return params
resolved = dict(params)
for param_key, ref in mapped.items():
if str(param_key) in _BLOCKED_PIN_KEYS or not isinstance(ref, dict):
continue
src_id = str(ref.get("sourceId") or ref.get("source_id") or "")
src_handle = str(ref.get("sourceHandle") or ref.get("source_handle") or "result")
node_out = node_outputs_store.get(src_id)
if not isinstance(node_out, dict):
continue
for candidate in (src_handle, "result", "val"):
if candidate in node_out:
resolved[param_key] = node_out[candidate]
break
return resolved
def init_queued_node_results(
execution_order: List[str],
node_map: Dict[str, Dict[str, Any]],
run_id: str,
graph_revision: str,
created_ts: str,
) -> Dict[str, Dict[str, Any]]:
"""Initialize queued telemetry records and emit queued events for scheduled nodes."""
node_results: Dict[str, Dict[str, Any]] = {}
for nid in execution_order:
n_node = node_map[nid]
d_init = n_node.get("data") or {}
ntype_init = str(n_node.get("node_type") or n_node.get("type") or d_init.get("node_type") or d_init.get("nodeType") or "").lower()
lbl_init = d_init.get("label") or n_node.get("label") or ntype_init or nid
node_results[nid] = {
"run_id": run_id, "node_id": nid, "node_type": ntype_init, "label": lbl_init,
"status": "queued", "config_state": "valid", "exec_state": "queued", "freshness": "no_result",
"inputs": {}, "outputs": {}, "output_ports": {}, "duration_ms": 0.0, "retries_attempted": 0,
"error": None, "error_semantics": None, "skip_reason": None, "stale_reason": None,
"timestamp": created_ts,
}
GLOBAL_EVENT_STORE.record_event(
run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_QUEUED,
node_id=nid, status="queued", message=f"Node '{lbl_init}' ({nid}) queued for execution.",
payload={"node_type": ntype_init},
)
return node_results
def build_graph_execution_order(
graph: Dict[str, Any], raw_nodes: List[Dict[str, Any]], raw_edges: List[Dict[str, Any]]
) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, List[Dict[str, Any]]], Dict[str, List[Dict[str, Any]]], List[str], Dict[str, Any]]:
"""Build node lookup, adjacency maps, and deterministic execution order."""
node_map = {n["id"]: n for n in raw_nodes if n.get("id")}
in_degrees: Dict[str, int] = {nid: 0 for nid in node_map}
adj_list: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map}
incoming_map: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map}
for e in raw_edges:
s, t = e.get("source") or e.get("from_node"), e.get("target") or e.get("to_node")
if s in node_map and t in node_map:
adj_list[s].append(e)
incoming_map[t].append(e)
in_degrees[t] += 1
exec_plan = build_execution_plan(graph)
if exec_plan.get("valid") and exec_plan.get("execution_order"):
execution_order: List[str] = exec_plan["execution_order"]
else:
queue = deque(sorted([nid for nid, deg in in_degrees.items() if deg == 0]))
execution_order = []
while queue:
curr = queue.popleft()
execution_order.append(curr)
for nbr in sorted([e.get("target") or e.get("to_node") for e in adj_list.get(curr, [])]):
in_degrees[nbr] -= 1
if in_degrees[nbr] == 0:
queue.append(nbr)
return node_map, adj_list, incoming_map, execution_order, exec_plan
def mark_run_cancelled(
execution_order: List[str], node_results: Dict[str, Dict[str, Any]], run_id: str, graph_revision: str
) -> None:
"""Mark queued/pending nodes cancelled and emit cancellation events."""
for rem_id in execution_order:
if node_results.get(rem_id, {}).get("exec_state") in ("queued", "idle", "pending", None):
rem_lbl = node_results[rem_id].get("label", rem_id)
node_results[rem_id].update({
"status": "cancelled",
"exec_state": "cancelled",
"freshness": "invalidated",
"error": "[CANCELLED] Node execution was cancelled by user request.",
"error_semantics": make_error_semantics(
code="ERR-CANCELLED",
what=f"Execution cancelled: {rem_lbl}",
node_id=rem_id,
node_label=rem_lbl,
why="Workflow run was cancelled before this node executed.",
how_to_fix="Rerun the workflow to execute all nodes.",
severity="info",
),
})
GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_CANCELLED, node_id=rem_id, status="cancelled", message=f"Node '{rem_id}' cancelled.")
GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_RUN_CANCELLED, status="cancelled", message=f"Workflow run '{run_id}' cancelled.")
def scrub_result_secrets(payload: Dict[str, Any], ctx: Optional[RunEnvContext] = None) -> Dict[str, Any]:
"""Redact workspace secrets and common token formats from execution result payloads."""
if not isinstance(payload, dict):
return payload
def _scrub_str(val: Any) -> Any:
if not isinstance(val, str):
return val
return redact_sensitive_text(redact_secrets_from_text(val, ctx))
def _scrub_any(val: Any) -> Any:
return redact_sensitive_data(redact_secrets_from_obj(val, ctx))
for key in ("stdout", "stderr", "error"):
if key in payload and payload[key]:
payload[key] = _scrub_str(payload[key])
if isinstance(payload.get("logs"), list):
payload["logs"] = [_scrub_str(item) for item in payload["logs"]]
for key in ("final_outputs", "node_results", "outputs", "node_result", "output_ports", "previous_outputs", "session_state", "inputs"):
if key in payload and payload[key] is not None:
payload[key] = _scrub_any(payload[key])
return payload
def scrub_session_secrets(session: Any, ctx: Optional[RunEnvContext] = None) -> None:
"""Scrub telemetry logs/results in-place and bind active secret tokens for safe persistence."""
if ctx and getattr(ctx, "secret_values", None):
existing = set(getattr(session, "_secret_values", ()) or ())
existing.update(ctx.secret_values)
session._secret_values = tuple(sorted(existing, key=len, reverse=True))
eff_ctx = ctx or (RunEnvContext(secret_values=session._secret_values) if getattr(session, "_secret_values", None) else None)
scrub_txt = lambda v: redact_sensitive_text(redact_secrets_from_text(v, eff_ctx)) if isinstance(v, str) else v
scrub_obj = lambda v: redact_sensitive_data(redact_secrets_from_obj(v, eff_ctx))
session.node_results = scrub_obj(session.node_results)
session.logs = [scrub_txt(item) for item in session.logs]
session.stdout_lines = [scrub_txt(item) for item in session.stdout_lines]
|