""" ALLI Workflow Executor Prep & Telemetry Hygiene Helper Handles cached upstream/pinned Studio test state for isolated single-node `Test step` runs and value-aware egress secret scrubbing across execution results. """ from collections import deque from typing import Any, Dict, List, Optional, Set, Tuple from language_core.execution_planner import build_execution_plan from language_core.executor.state_and_errors import make_error_semantics from language_core.runtime_events import ( EVENT_NODE_CANCELLED, EVENT_NODE_QUEUED, EVENT_RUN_CANCELLED, GLOBAL_EVENT_STORE, redact_sensitive_data, redact_sensitive_text, ) from runtime_environment.env_injection import ( RunEnvContext, redact_secrets_from_obj, redact_secrets_from_text, ) _BLOCKED_PIN_KEYS = frozenset({"enabled", "source", "__proto__", "constructor", "prototype"}) def _extract_pinned_map(node: Dict[str, Any], field_name: str, *, require_explicit_enable: bool) -> Dict[str, Any]: if not isinstance(node, dict): return {} data = node.get("data") if isinstance(node.get("data"), dict) else {} raw_pin = data.get(field_name) if isinstance(data.get(field_name), dict) else node.get(field_name) if not isinstance(raw_pin, dict): return {} if (require_explicit_enable and not raw_pin.get("enabled", True)) or ("enabled" in raw_pin and not raw_pin.get("enabled", True)): return {} inner = raw_pin.get("data") if isinstance(raw_pin.get("data"), dict) else raw_pin return {str(k): v for k, v in inner.items() if str(k) not in _BLOCKED_PIN_KEYS} def extract_node_pinned_input(node: Dict[str, Any]) -> Dict[str, Any]: """Read Studio test-only pinned input dict from node/data (never baked into codegen).""" return _extract_pinned_map(node, "_pinned_input", require_explicit_enable=False) def extract_node_pinned_output(node: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Read Studio test-only pinned output dict from node/data when explicitly enabled.""" clean = _extract_pinned_map(node, "_pinned_output", require_explicit_enable=True) return clean if clean else None def prepare_isolated_step_execution( *, execution_order: List[str], node_map: Dict[str, Dict[str, Any]], target_node_id: Optional[str], single_node_only: bool, cached_outputs: Optional[Dict[str, Dict[str, Any]]], pinned_inputs: Optional[Dict[str, Dict[str, Any]]], node_outputs_store: Dict[str, Dict[str, Any]], ) -> List[str]: """Seed upstream cached/pinned outputs ONLY for isolated single-node Studio `Test step` execution.""" if not single_node_only: return execution_order for nid, node in node_map.items(): if nid == target_node_id: continue pinned_out = extract_node_pinned_output(node) if pinned_out: node_outputs_store[nid] = dict(pinned_out) if isinstance(cached_outputs, dict): for nid, out_map in cached_outputs.items(): if str(nid) in node_map and isinstance(out_map, dict): node_outputs_store[str(nid)] = {str(k): v for k, v in out_map.items() if str(k) not in _BLOCKED_PIN_KEYS} if target_node_id and target_node_id in node_map: if isinstance(pinned_inputs, dict) and isinstance(pinned_inputs.get(target_node_id), dict): target_node = node_map[target_node_id] data = target_node.setdefault("data", {}) if isinstance(data, dict): clean_pins = {str(k): v for k, v in pinned_inputs[target_node_id].items() if str(k) not in _BLOCKED_PIN_KEYS} data["_pinned_input"] = {"enabled": True, "data": clean_pins} return [target_node_id] return execution_order def resolve_mapped_node_params( params: Dict[str, Any], data: Dict[str, Any], node_outputs_store: Dict[str, Dict[str, Any]], ) -> Dict[str, Any]: """Resolve Stage-2 pill-mapped `_mapped_params` references against live/cached upstream outputs.""" mapped = ( params.get("_mapped_params") or data.get("_mapped_params") or (data.get("custom") if isinstance(data.get("custom"), dict) else {}).get("_mapped_params") ) if not isinstance(mapped, dict) or not mapped: return params resolved = dict(params) for param_key, ref in mapped.items(): if str(param_key) in _BLOCKED_PIN_KEYS or not isinstance(ref, dict): continue src_id = str(ref.get("sourceId") or ref.get("source_id") or "") src_handle = str(ref.get("sourceHandle") or ref.get("source_handle") or "result") node_out = node_outputs_store.get(src_id) if not isinstance(node_out, dict): continue for candidate in (src_handle, "result", "val"): if candidate in node_out: resolved[param_key] = node_out[candidate] break return resolved def init_queued_node_results( execution_order: List[str], node_map: Dict[str, Dict[str, Any]], run_id: str, graph_revision: str, created_ts: str, ) -> Dict[str, Dict[str, Any]]: """Initialize queued telemetry records and emit queued events for scheduled nodes.""" node_results: Dict[str, Dict[str, Any]] = {} for nid in execution_order: n_node = node_map[nid] d_init = n_node.get("data") or {} ntype_init = str(n_node.get("node_type") or n_node.get("type") or d_init.get("node_type") or d_init.get("nodeType") or "").lower() lbl_init = d_init.get("label") or n_node.get("label") or ntype_init or nid node_results[nid] = { "run_id": run_id, "node_id": nid, "node_type": ntype_init, "label": lbl_init, "status": "queued", "config_state": "valid", "exec_state": "queued", "freshness": "no_result", "inputs": {}, "outputs": {}, "output_ports": {}, "duration_ms": 0.0, "retries_attempted": 0, "error": None, "error_semantics": None, "skip_reason": None, "stale_reason": None, "timestamp": created_ts, } GLOBAL_EVENT_STORE.record_event( run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_QUEUED, node_id=nid, status="queued", message=f"Node '{lbl_init}' ({nid}) queued for execution.", payload={"node_type": ntype_init}, ) return node_results def build_graph_execution_order( graph: Dict[str, Any], raw_nodes: List[Dict[str, Any]], raw_edges: List[Dict[str, Any]] ) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, List[Dict[str, Any]]], Dict[str, List[Dict[str, Any]]], List[str], Dict[str, Any]]: """Build node lookup, adjacency maps, and deterministic execution order.""" node_map = {n["id"]: n for n in raw_nodes if n.get("id")} in_degrees: Dict[str, int] = {nid: 0 for nid in node_map} adj_list: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map} incoming_map: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map} for e in raw_edges: s, t = e.get("source") or e.get("from_node"), e.get("target") or e.get("to_node") if s in node_map and t in node_map: adj_list[s].append(e) incoming_map[t].append(e) in_degrees[t] += 1 exec_plan = build_execution_plan(graph) if exec_plan.get("valid") and exec_plan.get("execution_order"): execution_order: List[str] = exec_plan["execution_order"] else: queue = deque(sorted([nid for nid, deg in in_degrees.items() if deg == 0])) execution_order = [] while queue: curr = queue.popleft() execution_order.append(curr) for nbr in sorted([e.get("target") or e.get("to_node") for e in adj_list.get(curr, [])]): in_degrees[nbr] -= 1 if in_degrees[nbr] == 0: queue.append(nbr) return node_map, adj_list, incoming_map, execution_order, exec_plan def mark_run_cancelled( execution_order: List[str], node_results: Dict[str, Dict[str, Any]], run_id: str, graph_revision: str ) -> None: """Mark queued/pending nodes cancelled and emit cancellation events.""" for rem_id in execution_order: if node_results.get(rem_id, {}).get("exec_state") in ("queued", "idle", "pending", None): rem_lbl = node_results[rem_id].get("label", rem_id) node_results[rem_id].update({ "status": "cancelled", "exec_state": "cancelled", "freshness": "invalidated", "error": "[CANCELLED] Node execution was cancelled by user request.", "error_semantics": make_error_semantics( code="ERR-CANCELLED", what=f"Execution cancelled: {rem_lbl}", node_id=rem_id, node_label=rem_lbl, why="Workflow run was cancelled before this node executed.", how_to_fix="Rerun the workflow to execute all nodes.", severity="info", ), }) GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_CANCELLED, node_id=rem_id, status="cancelled", message=f"Node '{rem_id}' cancelled.") GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_RUN_CANCELLED, status="cancelled", message=f"Workflow run '{run_id}' cancelled.") def scrub_result_secrets(payload: Dict[str, Any], ctx: Optional[RunEnvContext] = None) -> Dict[str, Any]: """Redact workspace secrets and common token formats from execution result payloads.""" if not isinstance(payload, dict): return payload def _scrub_str(val: Any) -> Any: if not isinstance(val, str): return val return redact_sensitive_text(redact_secrets_from_text(val, ctx)) def _scrub_any(val: Any) -> Any: return redact_sensitive_data(redact_secrets_from_obj(val, ctx)) for key in ("stdout", "stderr", "error"): if key in payload and payload[key]: payload[key] = _scrub_str(payload[key]) if isinstance(payload.get("logs"), list): payload["logs"] = [_scrub_str(item) for item in payload["logs"]] for key in ("final_outputs", "node_results", "outputs", "node_result", "output_ports", "previous_outputs", "session_state", "inputs"): if key in payload and payload[key] is not None: payload[key] = _scrub_any(payload[key]) return payload def scrub_session_secrets(session: Any, ctx: Optional[RunEnvContext] = None) -> None: """Scrub telemetry logs/results in-place and bind active secret tokens for safe persistence.""" if ctx and getattr(ctx, "secret_values", None): existing = set(getattr(session, "_secret_values", ()) or ()) existing.update(ctx.secret_values) session._secret_values = tuple(sorted(existing, key=len, reverse=True)) eff_ctx = ctx or (RunEnvContext(secret_values=session._secret_values) if getattr(session, "_secret_values", None) else None) scrub_txt = lambda v: redact_sensitive_text(redact_secrets_from_text(v, eff_ctx)) if isinstance(v, str) else v scrub_obj = lambda v: redact_sensitive_data(redact_secrets_from_obj(v, eff_ctx)) session.node_results = scrub_obj(session.node_results) session.logs = [scrub_txt(item) for item in session.logs] session.stdout_lines = [scrub_txt(item) for item in session.stdout_lines]