rsnarsna
feat(alli-backend): Wave-1 standardization, standalone sys.env node & vault crypto, and DEC-014 security hardening
8628c98 Download lambdas/codegen/emitters_files_data.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 15.2 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/lambdas/codegen/emitters_files_data.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/lambdas/codegen/emitters_files_data.py
-
curl -L -o emitters_files_data.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/lambdas/codegen/emitters_files_data.py
15.2 kB
| """ | |
| ALLI Codegen Subpackage: File I/O, JSON, CSV & Collection Data Structure Emitters. | |
| """ | |
| from typing import Any, Optional, Tuple | |
| from language_core.codegen_contracts import format_literal | |
| from language_core.nodes.loader import parse_safe_literal_or_json | |
| from lambdas.codegen.context import NodeCodegenCtx | |
| def _eval_literal_default(raw: Any) -> Any: | |
| if isinstance(raw, str): | |
| ok, parsed = parse_safe_literal_or_json(raw) | |
| return parsed if ok else raw | |
| return raw | |
| def emit_file_and_structured_node(nctx: NodeCodegenCtx) -> Tuple[bool, Optional[str]]: | |
| raw_type, node_id, safe_id, params, cg = nctx.raw_type, nctx.node_id, nctx.safe_id, nctx.params, nctx.cg | |
| lines, ovm = cg.code_body_lines, cg.output_var_map | |
| if raw_type in ("file.read", "io.file_read") or (raw_type == "sys.file" and str(params.get("operation", "read")).lower() != "write"): | |
| cg.discovered_modules.add("os") | |
| path_expr = nctx.resolve_input(["file_path", "path"], format_literal(params.get("path") or params.get("file_path") or "data.txt")) | |
| from lambdas.codegen.wave1_core_emitters import sanitize_encoding_name | |
| enc, mode_val = sanitize_encoding_name(params.get("encoding")), params.get("mode", "text") | |
| nv, bv, sv = f"content_{safe_id}", f"bytes_{safe_id}", f"success_{safe_id}" | |
| if mode_val == "binary": | |
| lines.extend([f" with open({path_expr}, 'rb') as _f_{safe_id}:", f" {nv} = _f_{safe_id}.read()"]) | |
| else: | |
| lines.append(f" with open({path_expr}, 'r', encoding='{enc}') as _f_{safe_id}:") | |
| if mode_val == "lines": | |
| lines.append(f" {nv} = _f_{safe_id}.readlines()") | |
| elif mode_val == "json": | |
| cg.discovered_modules.add("json") | |
| lines.append(f" {nv} = json.load(_f_{safe_id})") | |
| else: | |
| lines.append(f" {nv} = _f_{safe_id}.read()") | |
| lines.extend([f" {bv} = len({nv})", f" {sv} = True"]) | |
| ovm.update({ | |
| f"{node_id}:content": nv, f"{node_id}:byte_count": bv, f"{node_id}:bytes": bv, | |
| f"{node_id}:lines": f"({nv}.splitlines() if isinstance({nv}, str) else {nv})", | |
| f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": nv, node_id: nv, | |
| }) | |
| return True, nv | |
| if raw_type in ("file.write", "io.file_write", "sys.file"): | |
| cg.discovered_modules.add("os") | |
| raw_path_param = params.get("file_path") or params.get("path", "output.txt") | |
| if isinstance(raw_path_param, str) and (".." in raw_path_param.replace("\\", "/").split("/") or raw_path_param.startswith(("/", "\\")) or (len(raw_path_param) > 1 and raw_path_param[1] == ":")): | |
| cg.diagnostics.append({"code": "SEC-001", "severity": "error", "message": f"Unsafe path '{raw_path_param}' rejected by sandbox boundary.", "node_id": node_id}) | |
| path_expr = nctx.resolve_input(["file_path", "path"], format_literal(raw_path_param)) | |
| raw_c_param = next((c for c in (params.get("content"), params.get("data"), params.get("text")) if c is not None), "") | |
| content_expr = nctx.resolve_input(["content", "data", "text"], format_literal(raw_c_param)) | |
| mode_opt = params.get("mode", "write") | |
| wmode = "a" if mode_opt == "append" else ("x" if mode_opt == "exclusive" else "w") | |
| from lambdas.codegen.wave1_core_emitters import sanitize_encoding_name | |
| enc = sanitize_encoding_name(params.get("encoding")) | |
| bv, pr, sv = f"written_{safe_id}", f"path_{safe_id}", f"success_{safe_id}" | |
| if bool(params.get("create_dirs", True)): | |
| lines.extend([f" if os.path.dirname({path_expr}):", f" os.makedirs(os.path.dirname({path_expr}), exist_ok=True)"]) | |
| lines.extend([ | |
| f" with open({path_expr}, '{wmode}', encoding='{enc}') as _f_{safe_id}:", | |
| f" _f_{safe_id}.write(str({content_expr}))", | |
| f" {bv} = len(str({content_expr}).encode('{enc}'))", | |
| f" {pr} = {path_expr}", | |
| f" {sv} = True", | |
| ]) | |
| ovm.update({f"{node_id}:bytes_written": bv, f"{node_id}:file_path": pr, f"{node_id}:success": sv, f"{node_id}:then": "True", node_id: bv}) | |
| return True, bv | |
| if raw_type in ("json.parse", "json_parse", "json.loads"): | |
| cg.discovered_modules.add("json") | |
| text_expr = nctx.resolve_input(["text", "json_str", "raw_json"], format_literal(params.get("text", "{}"))) | |
| strict = bool(params.get("strict", True)) | |
| dv, vv, ev = f"json_data_{safe_id}", f"is_valid_{safe_id}", f"err_{safe_id}" | |
| safe_txt = f"({text_expr}.text if hasattr({text_expr}, 'text') else (json.dumps({text_expr}) if isinstance({text_expr}, (dict, list)) else str({text_expr})))" | |
| if bool(params.get("fallback_to_none", False)): | |
| lines.extend([ | |
| " try:", f" {dv} = json.loads({safe_txt}, strict={strict})", | |
| f" {vv} = True", f" {ev} = None", | |
| f" except Exception as _e_{safe_id}:", f" {dv} = None", f" {vv} = False", f" {ev} = str(_e_{safe_id})", | |
| ]) | |
| else: | |
| lines.extend([f" {dv} = json.loads({safe_txt}, strict={strict})", f" {vv} = True", f" {ev} = None"]) | |
| ovm.update({f"{node_id}:data": dv, f"{node_id}:is_valid": vv, f"{node_id}:error": ev, f"{node_id}:then": "True", f"{node_id}:out": dv, node_id: dv}) | |
| return True, dv | |
| if raw_type in ("json.stringify", "json_stringify", "json.dumps"): | |
| cg.discovered_modules.add("json") | |
| data_expr = nctx.resolve_input(["data", "obj", "payload"], format_literal(params.get("data", {}))) | |
| iv = params.get("indent", 2) | |
| indent_arg = "None" if str(iv) in ("0", "none", "None") else str(iv) | |
| tv, bv, sv = f"json_str_{safe_id}", f"json_bytes_{safe_id}", f"success_{safe_id}" | |
| lines.extend([ | |
| f" {tv} = json.dumps({data_expr}, indent={indent_arg}, sort_keys={bool(params.get('sort_keys', False))}, ensure_ascii={bool(params.get('ensure_ascii', False))}, default=str)", | |
| f" {bv} = len({tv}.encode('utf-8'))", | |
| f" {sv} = True", | |
| ]) | |
| ovm.update({f"{node_id}:text": tv, f"{node_id}:byte_size": bv, f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": tv, node_id: tv}) | |
| return True, tv | |
| if raw_type in ("csv.parse", "csv_parse", "csv.read"): | |
| cg.discovered_modules.update(("csv", "io")) | |
| text_expr = nctx.resolve_input(["text", "csv_data", "raw_csv"], format_literal(params.get("text", ""))) | |
| delim = "\t" if params.get("delimiter", ",") == "\\t" else params.get("delimiter", ",") | |
| has_hdr, trim_ws = bool(params.get("has_header", True)), bool(params.get("trim_whitespace", True)) | |
| rv, hv, cv, sv, rdr, fv = f"csv_rows_{safe_id}", f"csv_headers_{safe_id}", f"csv_count_{safe_id}", f"success_{safe_id}", f"_reader_{safe_id}", f"_f_{safe_id}" | |
| lines.append(f" {fv} = io.StringIO(str({text_expr}))") | |
| if has_hdr: | |
| lines.extend([f" {rdr} = csv.DictReader({fv}, delimiter={format_literal(delim)})", f" {hv} = list({rdr}.fieldnames or [])"]) | |
| lines.append(f" {rv} = [{{(k.strip() if isinstance(k, str) else k): (v.strip() if isinstance(v, str) else v) for k, v in r.items()}} for r in {rdr} if any(str(x).strip() for x in r.values())]" if trim_ws else f" {rv} = [dict(r) for r in {rdr} if any(str(x).strip() for x in r.values())]") | |
| else: | |
| lines.extend([f" {hv} = []", f" {rdr} = csv.reader({fv}, delimiter={format_literal(delim)})"]) | |
| lines.append(f" {rv} = [[(v.strip() if isinstance(v, str) else v) for v in r] for r in {rdr} if r and any(str(x).strip() for x in r)]" if trim_ws else f" {rv} = [list(r) for r in {rdr} if r and any(str(x).strip() for x in r)]") | |
| lines.extend([f" {cv} = len({rv})", f" {sv} = True"]) | |
| ovm.update({f"{node_id}:rows": rv, f"{node_id}:headers": hv, f"{node_id}:row_count": cv, f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": rv, node_id: rv}) | |
| return True, rv | |
| if raw_type in ("csv.stringify", "csv_stringify", "csv.write"): | |
| cg.discovered_modules.update(("csv", "io")) | |
| rows_default = _eval_literal_default(params.get("rows", [])) | |
| rows_expr = nctx.resolve_input(["rows", "data", "table"], repr(rows_default) if isinstance(rows_default, (list, tuple)) else format_literal(rows_default)) | |
| hdr_default = _eval_literal_default(params.get("headers")) if params.get("headers") is not None else None | |
| hdr_expr = nctx.resolve_input(["headers", "custom_headers"], repr(hdr_default) if isinstance(hdr_default, list) else "None") | |
| delim = "\t" if params.get("delimiter", ",") == "\\t" else params.get("delimiter", ",") | |
| inc_hdr, term = bool(params.get("include_header", True)), params.get("line_terminator", "\n") | |
| tv, cv, bv, sv, fv, wv = f"csv_text_{safe_id}", f"csv_count_{safe_id}", f"csv_bytes_{safe_id}", f"success_{safe_id}", f"_f_{safe_id}", f"_w_{safe_id}" | |
| if not rows_expr.strip().startswith(("[", "(")): | |
| cg.discovered_modules.add("ast") | |
| raw_rows_expr = f"(ast.literal_eval({rows_expr}) if isinstance({rows_expr}, str) else {rows_expr})" | |
| else: | |
| raw_rows_expr = rows_expr | |
| lines.extend([ | |
| f" {fv} = io.StringIO()", | |
| f" _rows_in_{safe_id} = {raw_rows_expr}", | |
| f" _rows_{safe_id} = _rows_in_{safe_id} if isinstance(_rows_in_{safe_id}, (list, tuple)) else [_rows_in_{safe_id}]", | |
| f" _hdr_{safe_id} = {hdr_expr}", | |
| f" if _rows_{safe_id} and isinstance(_rows_{safe_id}[0], dict):", | |
| f" _fields_{safe_id} = list(_hdr_{safe_id}) if _hdr_{safe_id} and isinstance(_hdr_{safe_id}, list) else list(_rows_{safe_id}[0].keys())", | |
| f" {wv} = csv.DictWriter({fv}, fieldnames=_fields_{safe_id}, delimiter={format_literal(delim)}, lineterminator={format_literal(term)}, extrasaction='ignore')", | |
| ]) | |
| if inc_hdr: | |
| lines.append(f" {wv}.writeheader()") | |
| lines.extend([ | |
| f" for _r in _rows_{safe_id}:", | |
| " if isinstance(_r, dict):", | |
| f" {wv}.writerow(_r)", | |
| " else:", | |
| f" {wv} = csv.writer({fv}, delimiter={format_literal(delim)}, lineterminator={format_literal(term)})", | |
| ]) | |
| if inc_hdr: | |
| lines.extend([f" if _hdr_{safe_id} and isinstance(_hdr_{safe_id}, list):", f" {wv}.writerow(_hdr_{safe_id})"]) | |
| lines.extend([ | |
| f" for _r in _rows_{safe_id}:", | |
| " if isinstance(_r, (list, tuple)):", | |
| f" {wv}.writerow(_r)", | |
| " else:", | |
| f" {wv}.writerow([_r])", | |
| f" {tv} = {fv}.getvalue()", | |
| f" {cv} = len(_rows_{safe_id})", | |
| f" {bv} = len({tv}.encode('utf-8'))", | |
| f" {sv} = True", | |
| ]) | |
| ovm.update({f"{node_id}:text": tv, f"{node_id}:row_count": cv, f"{node_id}:byte_size": bv, f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": tv, node_id: tv}) | |
| return True, tv | |
| return False, None | |
| def emit_collection_node(nctx: NodeCodegenCtx) -> Tuple[bool, Optional[str]]: | |
| raw_type, node_id, safe_id, params, cg = nctx.raw_type, nctx.node_id, nctx.safe_id, nctx.params, nctx.cg | |
| lines, ovm = cg.code_body_lines, cg.output_var_map | |
| if raw_type in ("data.list", "list"): | |
| items_expr = nctx.resolve_input("items", _eval_literal_default(params.get("initial_items", "[]"))) | |
| nv = f"list_{safe_id}" | |
| lines.append(f" {nv} = {items_expr}") | |
| ovm.update({f"{node_id}:result": nv, f"{node_id}:list_out": nv, f"{node_id}:out": nv, f"{node_id}:length": f"len({nv})", f"{node_id}:count": f"len({nv})", node_id: nv}) | |
| return True, nv | |
| if raw_type in ("data.dict", "dict"): | |
| raw_init = _eval_literal_default(params.get("initial_pairs") or params.get("initial_dict", "{}")) | |
| connected = nctx.resolve_input("dict_in", "None") | |
| dict_expr = connected if connected != "None" else (repr(raw_init) if isinstance(raw_init, dict) else format_literal(raw_init)) | |
| nv = f"dict_{safe_id}" | |
| lines.append(f" {nv} = {dict_expr}") | |
| ovm.update({f"{node_id}:result": nv, f"{node_id}:dict_out": nv, f"{node_id}:out": nv, node_id: nv}) | |
| return True, nv | |
| if raw_type in ("data.tuple", "tuple"): | |
| has_items = any(k in params or k in nctx.node_inputs for k in ("item_1", "item_2", "item_3")) or any( | |
| str(inc.get("target_handle") or inc.get("targetHandle") or "").lower() in ("item_1", "item_2", "item_3") for inc in nctx.incoming | |
| ) | |
| nv = f"tuple_{safe_id}" | |
| if has_items and "items" not in params and "items" not in nctx.node_inputs: | |
| i1 = nctx.resolve_input("item_1", format_literal(params.get("item_1", 1))) | |
| i2 = nctx.resolve_input("item_2", format_literal(params.get("item_2", 2))) | |
| i3 = nctx.resolve_input("item_3", format_literal(params.get("item_3", 3))) | |
| lines.append(f" {nv} = ({i1}, {i2}, {i3})") | |
| else: | |
| parsed_def = _eval_literal_default(params.get("items") or params.get("initial_items", "()")) | |
| tuple_expr = nctx.resolve_input("items", repr(parsed_def) if isinstance(parsed_def, (list, tuple, set)) else format_literal(parsed_def)) | |
| if not tuple_expr.strip().startswith(("[", "(", "{")): | |
| cg.discovered_modules.add("ast") | |
| tuple_expr = f"(ast.literal_eval({tuple_expr}) if isinstance({tuple_expr}, str) else {tuple_expr})" | |
| lines.append(f" {nv} = tuple({tuple_expr})") | |
| ovm.update({f"{node_id}:result": nv, f"{node_id}:tuple_out": nv, f"{node_id}:out": nv, f"{node_id}:count": f"len({nv})", node_id: nv}) | |
| return True, nv | |
| if raw_type in ("data.set", "set"): | |
| raw_set_def = _eval_literal_default(params.get("initial_items") or params.get("initial_elements", "[]")) | |
| elements_expr = nctx.resolve_input(["set_a", "items", "initial_elements"], repr(raw_set_def) if isinstance(raw_set_def, (list, tuple, set)) else format_literal(raw_set_def)) | |
| if not elements_expr.strip().startswith(("[", "(", "{")): | |
| cg.discovered_modules.add("ast") | |
| elements_expr = f"(ast.literal_eval({elements_expr}) if isinstance({elements_expr}, str) else {elements_expr})" | |
| nv = f"set_{safe_id}" | |
| lines.append(f" {nv} = set({elements_expr})") | |
| add_expr = nctx.resolve_input(["add_item", "element"], "None") | |
| if add_expr != "None": | |
| lines.append(f" {nv}.add({add_expr})") | |
| ovm.update({f"{node_id}:result": nv, f"{node_id}:set_out": nv, f"{node_id}:out": nv, f"{node_id}:count": f"len({nv})", f"{node_id}:cardinality": f"len({nv})", node_id: nv}) | |
| return True, nv | |
| return False, None | |