rsnarsna
feat(alli-backend): Wave-1 standardization, standalone sys.env node & vault crypto, and DEC-014 security hardening
8628c98 Download language_core/executor/run_prep.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/language_core/executor/run_prep.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/language_core/executor/run_prep.py
-
curl -L -o run_prep.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/language_core/executor/run_prep.py
11.5 kB
| """ | |
| ALLI Workflow Executor Prep & Telemetry Hygiene Helper | |
| Handles cached upstream/pinned Studio test state for isolated single-node `Test step` | |
| runs and value-aware egress secret scrubbing across execution results. | |
| """ | |
| from collections import deque | |
| from typing import Any, Dict, List, Optional, Set, Tuple | |
| from language_core.execution_planner import build_execution_plan | |
| from language_core.executor.state_and_errors import make_error_semantics | |
| from language_core.runtime_events import ( | |
| EVENT_NODE_CANCELLED, | |
| EVENT_NODE_QUEUED, | |
| EVENT_RUN_CANCELLED, | |
| GLOBAL_EVENT_STORE, | |
| redact_sensitive_data, | |
| redact_sensitive_text, | |
| ) | |
| from runtime_environment.env_injection import ( | |
| RunEnvContext, | |
| redact_secrets_from_obj, | |
| redact_secrets_from_text, | |
| ) | |
| _BLOCKED_PIN_KEYS = frozenset({"enabled", "source", "__proto__", "constructor", "prototype"}) | |
| def _extract_pinned_map(node: Dict[str, Any], field_name: str, *, require_explicit_enable: bool) -> Dict[str, Any]: | |
| if not isinstance(node, dict): | |
| return {} | |
| data = node.get("data") if isinstance(node.get("data"), dict) else {} | |
| raw_pin = data.get(field_name) if isinstance(data.get(field_name), dict) else node.get(field_name) | |
| if not isinstance(raw_pin, dict): | |
| return {} | |
| if (require_explicit_enable and not raw_pin.get("enabled", True)) or ("enabled" in raw_pin and not raw_pin.get("enabled", True)): | |
| return {} | |
| inner = raw_pin.get("data") if isinstance(raw_pin.get("data"), dict) else raw_pin | |
| return {str(k): v for k, v in inner.items() if str(k) not in _BLOCKED_PIN_KEYS} | |
| def extract_node_pinned_input(node: Dict[str, Any]) -> Dict[str, Any]: | |
| """Read Studio test-only pinned input dict from node/data (never baked into codegen).""" | |
| return _extract_pinned_map(node, "_pinned_input", require_explicit_enable=False) | |
| def extract_node_pinned_output(node: Dict[str, Any]) -> Optional[Dict[str, Any]]: | |
| """Read Studio test-only pinned output dict from node/data when explicitly enabled.""" | |
| clean = _extract_pinned_map(node, "_pinned_output", require_explicit_enable=True) | |
| return clean if clean else None | |
| def prepare_isolated_step_execution( | |
| *, | |
| execution_order: List[str], | |
| node_map: Dict[str, Dict[str, Any]], | |
| target_node_id: Optional[str], | |
| single_node_only: bool, | |
| cached_outputs: Optional[Dict[str, Dict[str, Any]]], | |
| pinned_inputs: Optional[Dict[str, Dict[str, Any]]], | |
| node_outputs_store: Dict[str, Dict[str, Any]], | |
| ) -> List[str]: | |
| """Seed upstream cached/pinned outputs ONLY for isolated single-node Studio `Test step` execution.""" | |
| if not single_node_only: | |
| return execution_order | |
| for nid, node in node_map.items(): | |
| if nid == target_node_id: | |
| continue | |
| pinned_out = extract_node_pinned_output(node) | |
| if pinned_out: | |
| node_outputs_store[nid] = dict(pinned_out) | |
| if isinstance(cached_outputs, dict): | |
| for nid, out_map in cached_outputs.items(): | |
| if str(nid) in node_map and isinstance(out_map, dict): | |
| node_outputs_store[str(nid)] = {str(k): v for k, v in out_map.items() if str(k) not in _BLOCKED_PIN_KEYS} | |
| if target_node_id and target_node_id in node_map: | |
| if isinstance(pinned_inputs, dict) and isinstance(pinned_inputs.get(target_node_id), dict): | |
| target_node = node_map[target_node_id] | |
| data = target_node.setdefault("data", {}) | |
| if isinstance(data, dict): | |
| clean_pins = {str(k): v for k, v in pinned_inputs[target_node_id].items() if str(k) not in _BLOCKED_PIN_KEYS} | |
| data["_pinned_input"] = {"enabled": True, "data": clean_pins} | |
| return [target_node_id] | |
| return execution_order | |
| def resolve_mapped_node_params( | |
| params: Dict[str, Any], | |
| data: Dict[str, Any], | |
| node_outputs_store: Dict[str, Dict[str, Any]], | |
| ) -> Dict[str, Any]: | |
| """Resolve Stage-2 pill-mapped `_mapped_params` references against live/cached upstream outputs.""" | |
| mapped = ( | |
| params.get("_mapped_params") | |
| or data.get("_mapped_params") | |
| or (data.get("custom") if isinstance(data.get("custom"), dict) else {}).get("_mapped_params") | |
| ) | |
| if not isinstance(mapped, dict) or not mapped: | |
| return params | |
| resolved = dict(params) | |
| for param_key, ref in mapped.items(): | |
| if str(param_key) in _BLOCKED_PIN_KEYS or not isinstance(ref, dict): | |
| continue | |
| src_id = str(ref.get("sourceId") or ref.get("source_id") or "") | |
| src_handle = str(ref.get("sourceHandle") or ref.get("source_handle") or "result") | |
| node_out = node_outputs_store.get(src_id) | |
| if not isinstance(node_out, dict): | |
| continue | |
| for candidate in (src_handle, "result", "val"): | |
| if candidate in node_out: | |
| resolved[param_key] = node_out[candidate] | |
| break | |
| return resolved | |
| def init_queued_node_results( | |
| execution_order: List[str], | |
| node_map: Dict[str, Dict[str, Any]], | |
| run_id: str, | |
| graph_revision: str, | |
| created_ts: str, | |
| ) -> Dict[str, Dict[str, Any]]: | |
| """Initialize queued telemetry records and emit queued events for scheduled nodes.""" | |
| node_results: Dict[str, Dict[str, Any]] = {} | |
| for nid in execution_order: | |
| n_node = node_map[nid] | |
| d_init = n_node.get("data") or {} | |
| ntype_init = str(n_node.get("node_type") or n_node.get("type") or d_init.get("node_type") or d_init.get("nodeType") or "").lower() | |
| lbl_init = d_init.get("label") or n_node.get("label") or ntype_init or nid | |
| node_results[nid] = { | |
| "run_id": run_id, "node_id": nid, "node_type": ntype_init, "label": lbl_init, | |
| "status": "queued", "config_state": "valid", "exec_state": "queued", "freshness": "no_result", | |
| "inputs": {}, "outputs": {}, "output_ports": {}, "duration_ms": 0.0, "retries_attempted": 0, | |
| "error": None, "error_semantics": None, "skip_reason": None, "stale_reason": None, | |
| "timestamp": created_ts, | |
| } | |
| GLOBAL_EVENT_STORE.record_event( | |
| run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_QUEUED, | |
| node_id=nid, status="queued", message=f"Node '{lbl_init}' ({nid}) queued for execution.", | |
| payload={"node_type": ntype_init}, | |
| ) | |
| return node_results | |
| def build_graph_execution_order( | |
| graph: Dict[str, Any], raw_nodes: List[Dict[str, Any]], raw_edges: List[Dict[str, Any]] | |
| ) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, List[Dict[str, Any]]], Dict[str, List[Dict[str, Any]]], List[str], Dict[str, Any]]: | |
| """Build node lookup, adjacency maps, and deterministic execution order.""" | |
| node_map = {n["id"]: n for n in raw_nodes if n.get("id")} | |
| in_degrees: Dict[str, int] = {nid: 0 for nid in node_map} | |
| adj_list: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map} | |
| incoming_map: Dict[str, List[Dict[str, Any]]] = {nid: [] for nid in node_map} | |
| for e in raw_edges: | |
| s, t = e.get("source") or e.get("from_node"), e.get("target") or e.get("to_node") | |
| if s in node_map and t in node_map: | |
| adj_list[s].append(e) | |
| incoming_map[t].append(e) | |
| in_degrees[t] += 1 | |
| exec_plan = build_execution_plan(graph) | |
| if exec_plan.get("valid") and exec_plan.get("execution_order"): | |
| execution_order: List[str] = exec_plan["execution_order"] | |
| else: | |
| queue = deque(sorted([nid for nid, deg in in_degrees.items() if deg == 0])) | |
| execution_order = [] | |
| while queue: | |
| curr = queue.popleft() | |
| execution_order.append(curr) | |
| for nbr in sorted([e.get("target") or e.get("to_node") for e in adj_list.get(curr, [])]): | |
| in_degrees[nbr] -= 1 | |
| if in_degrees[nbr] == 0: | |
| queue.append(nbr) | |
| return node_map, adj_list, incoming_map, execution_order, exec_plan | |
| def mark_run_cancelled( | |
| execution_order: List[str], node_results: Dict[str, Dict[str, Any]], run_id: str, graph_revision: str | |
| ) -> None: | |
| """Mark queued/pending nodes cancelled and emit cancellation events.""" | |
| for rem_id in execution_order: | |
| if node_results.get(rem_id, {}).get("exec_state") in ("queued", "idle", "pending", None): | |
| rem_lbl = node_results[rem_id].get("label", rem_id) | |
| node_results[rem_id].update({ | |
| "status": "cancelled", | |
| "exec_state": "cancelled", | |
| "freshness": "invalidated", | |
| "error": "[CANCELLED] Node execution was cancelled by user request.", | |
| "error_semantics": make_error_semantics( | |
| code="ERR-CANCELLED", | |
| what=f"Execution cancelled: {rem_lbl}", | |
| node_id=rem_id, | |
| node_label=rem_lbl, | |
| why="Workflow run was cancelled before this node executed.", | |
| how_to_fix="Rerun the workflow to execute all nodes.", | |
| severity="info", | |
| ), | |
| }) | |
| GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_NODE_CANCELLED, node_id=rem_id, status="cancelled", message=f"Node '{rem_id}' cancelled.") | |
| GLOBAL_EVENT_STORE.record_event(run_id=run_id, graph_revision=graph_revision, event_type=EVENT_RUN_CANCELLED, status="cancelled", message=f"Workflow run '{run_id}' cancelled.") | |
| def scrub_result_secrets(payload: Dict[str, Any], ctx: Optional[RunEnvContext] = None) -> Dict[str, Any]: | |
| """Redact workspace secrets and common token formats from execution result payloads.""" | |
| if not isinstance(payload, dict): | |
| return payload | |
| def _scrub_str(val: Any) -> Any: | |
| if not isinstance(val, str): | |
| return val | |
| return redact_sensitive_text(redact_secrets_from_text(val, ctx)) | |
| def _scrub_any(val: Any) -> Any: | |
| return redact_sensitive_data(redact_secrets_from_obj(val, ctx)) | |
| for key in ("stdout", "stderr", "error"): | |
| if key in payload and payload[key]: | |
| payload[key] = _scrub_str(payload[key]) | |
| if isinstance(payload.get("logs"), list): | |
| payload["logs"] = [_scrub_str(item) for item in payload["logs"]] | |
| for key in ("final_outputs", "node_results", "outputs", "node_result", "output_ports", "previous_outputs", "session_state", "inputs"): | |
| if key in payload and payload[key] is not None: | |
| payload[key] = _scrub_any(payload[key]) | |
| return payload | |
| def scrub_session_secrets(session: Any, ctx: Optional[RunEnvContext] = None) -> None: | |
| """Scrub telemetry logs/results in-place and bind active secret tokens for safe persistence.""" | |
| if ctx and getattr(ctx, "secret_values", None): | |
| existing = set(getattr(session, "_secret_values", ()) or ()) | |
| existing.update(ctx.secret_values) | |
| session._secret_values = tuple(sorted(existing, key=len, reverse=True)) | |
| eff_ctx = ctx or (RunEnvContext(secret_values=session._secret_values) if getattr(session, "_secret_values", None) else None) | |
| scrub_txt = lambda v: redact_sensitive_text(redact_secrets_from_text(v, eff_ctx)) if isinstance(v, str) else v | |
| scrub_obj = lambda v: redact_sensitive_data(redact_secrets_from_obj(v, eff_ctx)) | |
| session.node_results = scrub_obj(session.node_results) | |
| session.logs = [scrub_txt(item) for item in session.logs] | |
| session.stdout_lines = [scrub_txt(item) for item in session.stdout_lines] | |