auto / language_core /executor /run_prep.py
rsnarsna
feat(alli-backend): Wave-1 standardization, standalone sys.env node & vault crypto, and DEC-014 security hardening
8628c98
Raw History Blame Contribute Delete
11.5 kB
"""
ALLI Workflow Executor Prep & Telemetry Hygiene Helper
Handles cached upstream/pinned Studio test state for isolated single-node `Test step`
runs and value-aware egress secret scrubbing across execution results.
"""
from collections import deque
from typing import Any, Dict, List, Optional, Set, Tuple
from language_core.execution_planner import build_execution_plan
from language_core.executor.state_and_errors import make_error_semantics
from language_core.runtime_events import (
EVENT_NODE_CANCELLED,
EVENT_NODE_QUEUED,
EVENT_RUN_CANCELLED,
GLOBAL_EVENT_STORE,
redact_sensitive_data,
redact_sensitive_text,
)
from runtime_environment.env_injection import (
RunEnvContext,
redact_secrets_from_obj,
redact_secrets_from_text,
)
_BLOCKED_PIN_KEYS = frozenset({"enabled", "source", "__proto__", "constructor", "prototype"})
def _extract_pinned_map(node: Dict[str, Any], field_name: str, *, require_explicit_enable: bool) -> Dict[str, Any]:
if not isinstance(node, dict):
return {}
data = node.get("data") if isinstance(node.get("data"), dict) else {}
raw_pin = data.get(field_name) if isinstance(data.get(field_name), dict) else node.get(field_name)
if not isinstance(raw_pin, dict):
return {}
if (require_explicit_enable and not raw_pin.get("enabled", True)) or ("enabled" in raw_pin and not raw_pin.get("enabled", True)):
return {}
inner = raw_pin.get("data") if isinstance(raw_pin.get("data"), dict) else raw_pin
return {str(k): v for k, v in inner.items() if str(k) not in _BLOCKED_PIN_KEYS}
def extract_node_pinned_input(node: Dict[str, Any]) -> Dict[str, Any]:
"""Read Studio test-only pinned input dict from node/data (never baked into codegen)."""
return _extract_pinned_map(node, "_pinned_input", require_explicit_enable=False)
def extract_node_pinned_output(node: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Read Studio test-only pinned output dict from node/data when explicitly enabled."""
clean = _extract_pinned_map(node, "_pinned_output", require_explicit_enable=True)
return clean if clean else None
def prepare_isolated_step_execution(
*,
execution_order: List[str],
node_map: Dict[str, Dict[str, Any]],
target_node_id: Optional[str],
single_node_only: bool,
cached_outputs: Optional[Dict[str, Dict[str, Any]]],
pinned_inputs: Optional[Dict[str, Dict[str, Any]]],
node_outputs_store: Dict[str, Dict[str, Any]],
) -> List[str]:
"""Seed upstream cached/pinned outputs ONLY for isolated single-node Studio `Test step` execution."""
if not single_node_only:
return execution_order
for nid, node in node_map.items():
if nid == target_node_id:
continue
pinned_out = extract_node_pinned_output(node)
if pinned_out:
node_outputs_store[nid] = dict(pinned_out)
if isinstance(cached_outputs, dict):
for nid, out_map in cached_outputs.items():
if str(nid) in node_map and isinstance(out_map, dict):
node_outputs_store[str(nid)] = {str(k): v for k, v in out_map.items() if str(k) not in _BLOCKED_PIN_KEYS}
if target_node_id and target_node_id in node_map:
if isinstance(pinned_inputs, dict) and isinstance(pinned_inputs.get(target_node_id), dict):
target_node = node_map[target_node_id]
data = target_node.setdefault("data", {})
if isinstance(data, dict):
clean_pins = {str(k): v for k, v in pinned_inputs[target_node_id].items() if str(k) not in _BLOCKED_PIN_KEYS}
data["_pinned_input"] = {"enabled": True, "data": clean_pins}
return [target_node_id]
return execution_order
def resolve_mapped_node_params(
params: Dict[str, Any],
data: Dict[str, Any],
node_outputs_store: Dict[str, Dict[str, Any]],
) -> Dict[str, Any]:
"""Resolve Stage-2 pill-mapped `_mapped_params` references against live/cached upstream outputs."""
mapped = (
params.get("_mapped_params")
or data.get("_mapped_params")
or (data.get("custom") if isinstance(data.get("custom"), dict) else {}).get("_mapped_params")
)
if not isinstance(mapped, dict) or not mapped:
return params
resolved = dict(params)
for param_key, ref in mapped.items():
if str(param_key) in _BLOCKED_PIN_KEYS or not isinstance(ref, dict):
continue
src_id = str(ref.get("sourceId") or ref.get("source_id") or "")
src_handle = str(ref.get("sourceHandle") or ref.get("source_handle") or "result")
node_out = node_outputs_store.get(src_id)
if not isinstance(node_out, dict):
continue
for candidate in (src_handle, "result", "val"):
if candidate in node_out:
resolved[param_key] = node_out[candidate]
break
return resolved
def init_queued_node_results(
execution_order: List[str],
node_map: Dict[str, Dict[str, Any]],
run_id: str,
graph_revision: str,
created_ts: str,
) -> Dict[str, Dict[str, Any]]:
"""Initialize queued telemetry records and emit queued events for scheduled nodes."""
node_results: Dict[str, Dict[str, Any]] = {}
for nid in execution_order:
n_node = node_map[nid]
d_init = n_node.get("data") or {}
ntype_init = str(n_node.get("node_type") or n_node.get("type") or d_init.get("node_type") or d_init.get("nodeType") or "").lower()
lbl_init = d_init.get("label") or n_node.get("label") or ntype_init or nid
node_results[nid] = {
"run_id": run_id, "node_id": nid, "node_type": ntype_init, "label": lbl_init,
"status": "queued", "config_state": "valid", "exec_state": "queued", "freshness": "no_result",
"inputs": {}, "outputs": {}, "output_ports": {}, "duration_ms": 0.0, "retries_attempted": 0,
"error": None, "error_semantics": None, "skip_reason": None, "stale_reason": None,
"timestamp": created_ts,
}
GLOBAL_EVENT_STORE.record_event(
run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_QUEUED,
node_id=nid, status="queued", message=f"Node '{lbl_init}' ({nid}) queued for execution.",
payload={"node_type": ntype_init},
)
return node_results
def build_graph_execution_order(
graph: Dict[str, Any], raw_nodes: List[Dict[str, Any]], raw_edges: List[Dict[str, Any]]
) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, List[Dict[str, Any]]], Dict[str, List[Dict[str, Any]]], List[str], Dict[str, Any]]:
"""Build node lookup, adjacency maps, and deterministic execution order."""
node_map = {n["id"]: n for n in raw_nodes if n.get("id")}
in_degrees: Dict[str, int] = {nid: 0 for nid in node_map}
adj_list: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map}
incoming_map: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map}
for e in raw_edges:
s, t = e.get("source") or e.get("from_node"), e.get("target") or e.get("to_node")
if s in node_map and t in node_map:
adj_list[s].append(e)
incoming_map[t].append(e)
in_degrees[t] += 1
exec_plan = build_execution_plan(graph)
if exec_plan.get("valid") and exec_plan.get("execution_order"):
execution_order: List[str] = exec_plan["execution_order"]
else:
queue = deque(sorted([nid for nid, deg in in_degrees.items() if deg == 0]))
execution_order = []
while queue:
curr = queue.popleft()
execution_order.append(curr)
for nbr in sorted([e.get("target") or e.get("to_node") for e in adj_list.get(curr, [])]):
in_degrees[nbr] -= 1
if in_degrees[nbr] == 0:
queue.append(nbr)
return node_map, adj_list, incoming_map, execution_order, exec_plan
def mark_run_cancelled(
execution_order: List[str], node_results: Dict[str, Dict[str, Any]], run_id: str, graph_revision: str
) -> None:
"""Mark queued/pending nodes cancelled and emit cancellation events."""
for rem_id in execution_order:
if node_results.get(rem_id, {}).get("exec_state") in ("queued", "idle", "pending", None):
rem_lbl = node_results[rem_id].get("label", rem_id)
node_results[rem_id].update({
"status": "cancelled",
"exec_state": "cancelled",
"freshness": "invalidated",
"error": "[CANCELLED] Node execution was cancelled by user request.",
"error_semantics": make_error_semantics(
code="ERR-CANCELLED",
what=f"Execution cancelled: {rem_lbl}",
node_id=rem_id,
node_label=rem_lbl,
why="Workflow run was cancelled before this node executed.",
how_to_fix="Rerun the workflow to execute all nodes.",
severity="info",
),
})
GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_CANCELLED, node_id=rem_id, status="cancelled", message=f"Node '{rem_id}' cancelled.")
GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_RUN_CANCELLED, status="cancelled", message=f"Workflow run '{run_id}' cancelled.")
def scrub_result_secrets(payload: Dict[str, Any], ctx: Optional[RunEnvContext] = None) -> Dict[str, Any]:
"""Redact workspace secrets and common token formats from execution result payloads."""
if not isinstance(payload, dict):
return payload
def _scrub_str(val: Any) -> Any:
if not isinstance(val, str):
return val
return redact_sensitive_text(redact_secrets_from_text(val, ctx))
def _scrub_any(val: Any) -> Any:
return redact_sensitive_data(redact_secrets_from_obj(val, ctx))
for key in ("stdout", "stderr", "error"):
if key in payload and payload[key]:
payload[key] = _scrub_str(payload[key])
if isinstance(payload.get("logs"), list):
payload["logs"] = [_scrub_str(item) for item in payload["logs"]]
for key in ("final_outputs", "node_results", "outputs", "node_result", "output_ports", "previous_outputs", "session_state", "inputs"):
if key in payload and payload[key] is not None:
payload[key] = _scrub_any(payload[key])
return payload
def scrub_session_secrets(session: Any, ctx: Optional[RunEnvContext] = None) -> None:
"""Scrub telemetry logs/results in-place and bind active secret tokens for safe persistence."""
if ctx and getattr(ctx, "secret_values", None):
existing = set(getattr(session, "_secret_values", ()) or ())
existing.update(ctx.secret_values)
session._secret_values = tuple(sorted(existing, key=len, reverse=True))
eff_ctx = ctx or (RunEnvContext(secret_values=session._secret_values) if getattr(session, "_secret_values", None) else None)
scrub_txt = lambda v: redact_sensitive_text(redact_secrets_from_text(v, eff_ctx)) if isinstance(v, str) else v
scrub_obj = lambda v: redact_sensitive_data(redact_secrets_from_obj(v, eff_ctx))
session.node_results = scrub_obj(session.node_results)
session.logs = [scrub_txt(item) for item in session.logs]
session.stdout_lines = [scrub_txt(item) for item in session.stdout_lines]