File size: 15,164 Bytes
8628c98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 | """
ALLI Codegen Subpackage: File I/O, JSON, CSV & Collection Data Structure Emitters.
"""
from typing import Any, Optional, Tuple
from language_core.codegen_contracts import format_literal
from language_core.nodes.loader import parse_safe_literal_or_json
from lambdas.codegen.context import NodeCodegenCtx
def _eval_literal_default(raw: Any) -> Any:
if isinstance(raw, str):
ok, parsed = parse_safe_literal_or_json(raw)
return parsed if ok else raw
return raw
def emit_file_and_structured_node(nctx: NodeCodegenCtx) -> Tuple[bool, Optional[str]]:
raw_type, node_id, safe_id, params, cg = nctx.raw_type, nctx.node_id, nctx.safe_id, nctx.params, nctx.cg
lines, ovm = cg.code_body_lines, cg.output_var_map
if raw_type in ("file.read", "io.file_read") or (raw_type == "sys.file" and str(params.get("operation", "read")).lower() != "write"):
cg.discovered_modules.add("os")
path_expr = nctx.resolve_input(["file_path", "path"], format_literal(params.get("path") or params.get("file_path") or "data.txt"))
from lambdas.codegen.wave1_core_emitters import sanitize_encoding_name
enc, mode_val = sanitize_encoding_name(params.get("encoding")), params.get("mode", "text")
nv, bv, sv = f"content_{safe_id}", f"bytes_{safe_id}", f"success_{safe_id}"
if mode_val == "binary":
lines.extend([f" with open({path_expr}, 'rb') as _f_{safe_id}:", f" {nv} = _f_{safe_id}.read()"])
else:
lines.append(f" with open({path_expr}, 'r', encoding='{enc}') as _f_{safe_id}:")
if mode_val == "lines":
lines.append(f" {nv} = _f_{safe_id}.readlines()")
elif mode_val == "json":
cg.discovered_modules.add("json")
lines.append(f" {nv} = json.load(_f_{safe_id})")
else:
lines.append(f" {nv} = _f_{safe_id}.read()")
lines.extend([f" {bv} = len({nv})", f" {sv} = True"])
ovm.update({
f"{node_id}:content": nv, f"{node_id}:byte_count": bv, f"{node_id}:bytes": bv,
f"{node_id}:lines": f"({nv}.splitlines() if isinstance({nv}, str) else {nv})",
f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": nv, node_id: nv,
})
return True, nv
if raw_type in ("file.write", "io.file_write", "sys.file"):
cg.discovered_modules.add("os")
raw_path_param = params.get("file_path") or params.get("path", "output.txt")
if isinstance(raw_path_param, str) and (".." in raw_path_param.replace("\\", "/").split("/") or raw_path_param.startswith(("/", "\\")) or (len(raw_path_param) > 1 and raw_path_param[1] == ":")):
cg.diagnostics.append({"code": "SEC-001", "severity": "error", "message": f"Unsafe path '{raw_path_param}' rejected by sandbox boundary.", "node_id": node_id})
path_expr = nctx.resolve_input(["file_path", "path"], format_literal(raw_path_param))
raw_c_param = next((c for c in (params.get("content"), params.get("data"), params.get("text")) if c is not None), "")
content_expr = nctx.resolve_input(["content", "data", "text"], format_literal(raw_c_param))
mode_opt = params.get("mode", "write")
wmode = "a" if mode_opt == "append" else ("x" if mode_opt == "exclusive" else "w")
from lambdas.codegen.wave1_core_emitters import sanitize_encoding_name
enc = sanitize_encoding_name(params.get("encoding"))
bv, pr, sv = f"written_{safe_id}", f"path_{safe_id}", f"success_{safe_id}"
if bool(params.get("create_dirs", True)):
lines.extend([f" if os.path.dirname({path_expr}):", f" os.makedirs(os.path.dirname({path_expr}), exist_ok=True)"])
lines.extend([
f" with open({path_expr}, '{wmode}', encoding='{enc}') as _f_{safe_id}:",
f" _f_{safe_id}.write(str({content_expr}))",
f" {bv} = len(str({content_expr}).encode('{enc}'))",
f" {pr} = {path_expr}",
f" {sv} = True",
])
ovm.update({f"{node_id}:bytes_written": bv, f"{node_id}:file_path": pr, f"{node_id}:success": sv, f"{node_id}:then": "True", node_id: bv})
return True, bv
if raw_type in ("json.parse", "json_parse", "json.loads"):
cg.discovered_modules.add("json")
text_expr = nctx.resolve_input(["text", "json_str", "raw_json"], format_literal(params.get("text", "{}")))
strict = bool(params.get("strict", True))
dv, vv, ev = f"json_data_{safe_id}", f"is_valid_{safe_id}", f"err_{safe_id}"
safe_txt = f"({text_expr}.text if hasattr({text_expr}, 'text') else (json.dumps({text_expr}) if isinstance({text_expr}, (dict, list)) else str({text_expr})))"
if bool(params.get("fallback_to_none", False)):
lines.extend([
" try:", f" {dv} = json.loads({safe_txt}, strict={strict})",
f" {vv} = True", f" {ev} = None",
f" except Exception as _e_{safe_id}:", f" {dv} = None", f" {vv} = False", f" {ev} = str(_e_{safe_id})",
])
else:
lines.extend([f" {dv} = json.loads({safe_txt}, strict={strict})", f" {vv} = True", f" {ev} = None"])
ovm.update({f"{node_id}:data": dv, f"{node_id}:is_valid": vv, f"{node_id}:error": ev, f"{node_id}:then": "True", f"{node_id}:out": dv, node_id: dv})
return True, dv
if raw_type in ("json.stringify", "json_stringify", "json.dumps"):
cg.discovered_modules.add("json")
data_expr = nctx.resolve_input(["data", "obj", "payload"], format_literal(params.get("data", {})))
iv = params.get("indent", 2)
indent_arg = "None" if str(iv) in ("0", "none", "None") else str(iv)
tv, bv, sv = f"json_str_{safe_id}", f"json_bytes_{safe_id}", f"success_{safe_id}"
lines.extend([
f" {tv} = json.dumps({data_expr}, indent={indent_arg}, sort_keys={bool(params.get('sort_keys', False))}, ensure_ascii={bool(params.get('ensure_ascii', False))}, default=str)",
f" {bv} = len({tv}.encode('utf-8'))",
f" {sv} = True",
])
ovm.update({f"{node_id}:text": tv, f"{node_id}:byte_size": bv, f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": tv, node_id: tv})
return True, tv
if raw_type in ("csv.parse", "csv_parse", "csv.read"):
cg.discovered_modules.update(("csv", "io"))
text_expr = nctx.resolve_input(["text", "csv_data", "raw_csv"], format_literal(params.get("text", "")))
delim = "\t" if params.get("delimiter", ",") == "\\t" else params.get("delimiter", ",")
has_hdr, trim_ws = bool(params.get("has_header", True)), bool(params.get("trim_whitespace", True))
rv, hv, cv, sv, rdr, fv = f"csv_rows_{safe_id}", f"csv_headers_{safe_id}", f"csv_count_{safe_id}", f"success_{safe_id}", f"_reader_{safe_id}", f"_f_{safe_id}"
lines.append(f" {fv} = io.StringIO(str({text_expr}))")
if has_hdr:
lines.extend([f" {rdr} = csv.DictReader({fv}, delimiter={format_literal(delim)})", f" {hv} = list({rdr}.fieldnames or [])"])
lines.append(f" {rv} = [{{(k.strip() if isinstance(k, str) else k): (v.strip() if isinstance(v, str) else v) for k, v in r.items()}} for r in {rdr} if any(str(x).strip() for x in r.values())]" if trim_ws else f" {rv} = [dict(r) for r in {rdr} if any(str(x).strip() for x in r.values())]")
else:
lines.extend([f" {hv} = []", f" {rdr} = csv.reader({fv}, delimiter={format_literal(delim)})"])
lines.append(f" {rv} = [[(v.strip() if isinstance(v, str) else v) for v in r] for r in {rdr} if r and any(str(x).strip() for x in r)]" if trim_ws else f" {rv} = [list(r) for r in {rdr} if r and any(str(x).strip() for x in r)]")
lines.extend([f" {cv} = len({rv})", f" {sv} = True"])
ovm.update({f"{node_id}:rows": rv, f"{node_id}:headers": hv, f"{node_id}:row_count": cv, f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": rv, node_id: rv})
return True, rv
if raw_type in ("csv.stringify", "csv_stringify", "csv.write"):
cg.discovered_modules.update(("csv", "io"))
rows_default = _eval_literal_default(params.get("rows", []))
rows_expr = nctx.resolve_input(["rows", "data", "table"], repr(rows_default) if isinstance(rows_default, (list, tuple)) else format_literal(rows_default))
hdr_default = _eval_literal_default(params.get("headers")) if params.get("headers") is not None else None
hdr_expr = nctx.resolve_input(["headers", "custom_headers"], repr(hdr_default) if isinstance(hdr_default, list) else "None")
delim = "\t" if params.get("delimiter", ",") == "\\t" else params.get("delimiter", ",")
inc_hdr, term = bool(params.get("include_header", True)), params.get("line_terminator", "\n")
tv, cv, bv, sv, fv, wv = f"csv_text_{safe_id}", f"csv_count_{safe_id}", f"csv_bytes_{safe_id}", f"success_{safe_id}", f"_f_{safe_id}", f"_w_{safe_id}"
if not rows_expr.strip().startswith(("[", "(")):
cg.discovered_modules.add("ast")
raw_rows_expr = f"(ast.literal_eval({rows_expr}) if isinstance({rows_expr}, str) else {rows_expr})"
else:
raw_rows_expr = rows_expr
lines.extend([
f" {fv} = io.StringIO()",
f" _rows_in_{safe_id} = {raw_rows_expr}",
f" _rows_{safe_id} = _rows_in_{safe_id} if isinstance(_rows_in_{safe_id}, (list, tuple)) else [_rows_in_{safe_id}]",
f" _hdr_{safe_id} = {hdr_expr}",
f" if _rows_{safe_id} and isinstance(_rows_{safe_id}[0], dict):",
f" _fields_{safe_id} = list(_hdr_{safe_id}) if _hdr_{safe_id} and isinstance(_hdr_{safe_id}, list) else list(_rows_{safe_id}[0].keys())",
f" {wv} = csv.DictWriter({fv}, fieldnames=_fields_{safe_id}, delimiter={format_literal(delim)}, lineterminator={format_literal(term)}, extrasaction='ignore')",
])
if inc_hdr:
lines.append(f" {wv}.writeheader()")
lines.extend([
f" for _r in _rows_{safe_id}:",
" if isinstance(_r, dict):",
f" {wv}.writerow(_r)",
" else:",
f" {wv} = csv.writer({fv}, delimiter={format_literal(delim)}, lineterminator={format_literal(term)})",
])
if inc_hdr:
lines.extend([f" if _hdr_{safe_id} and isinstance(_hdr_{safe_id}, list):", f" {wv}.writerow(_hdr_{safe_id})"])
lines.extend([
f" for _r in _rows_{safe_id}:",
" if isinstance(_r, (list, tuple)):",
f" {wv}.writerow(_r)",
" else:",
f" {wv}.writerow([_r])",
f" {tv} = {fv}.getvalue()",
f" {cv} = len(_rows_{safe_id})",
f" {bv} = len({tv}.encode('utf-8'))",
f" {sv} = True",
])
ovm.update({f"{node_id}:text": tv, f"{node_id}:row_count": cv, f"{node_id}:byte_size": bv, f"{node_id}:success": sv, f"{node_id}:then": "True", f"{node_id}:out": tv, node_id: tv})
return True, tv
return False, None
def emit_collection_node(nctx: NodeCodegenCtx) -> Tuple[bool, Optional[str]]:
raw_type, node_id, safe_id, params, cg = nctx.raw_type, nctx.node_id, nctx.safe_id, nctx.params, nctx.cg
lines, ovm = cg.code_body_lines, cg.output_var_map
if raw_type in ("data.list", "list"):
items_expr = nctx.resolve_input("items", _eval_literal_default(params.get("initial_items", "[]")))
nv = f"list_{safe_id}"
lines.append(f" {nv} = {items_expr}")
ovm.update({f"{node_id}:result": nv, f"{node_id}:list_out": nv, f"{node_id}:out": nv, f"{node_id}:length": f"len({nv})", f"{node_id}:count": f"len({nv})", node_id: nv})
return True, nv
if raw_type in ("data.dict", "dict"):
raw_init = _eval_literal_default(params.get("initial_pairs") or params.get("initial_dict", "{}"))
connected = nctx.resolve_input("dict_in", "None")
dict_expr = connected if connected != "None" else (repr(raw_init) if isinstance(raw_init, dict) else format_literal(raw_init))
nv = f"dict_{safe_id}"
lines.append(f" {nv} = {dict_expr}")
ovm.update({f"{node_id}:result": nv, f"{node_id}:dict_out": nv, f"{node_id}:out": nv, node_id: nv})
return True, nv
if raw_type in ("data.tuple", "tuple"):
has_items = any(k in params or k in nctx.node_inputs for k in ("item_1", "item_2", "item_3")) or any(
str(inc.get("target_handle") or inc.get("targetHandle") or "").lower() in ("item_1", "item_2", "item_3") for inc in nctx.incoming
)
nv = f"tuple_{safe_id}"
if has_items and "items" not in params and "items" not in nctx.node_inputs:
i1 = nctx.resolve_input("item_1", format_literal(params.get("item_1", 1)))
i2 = nctx.resolve_input("item_2", format_literal(params.get("item_2", 2)))
i3 = nctx.resolve_input("item_3", format_literal(params.get("item_3", 3)))
lines.append(f" {nv} = ({i1}, {i2}, {i3})")
else:
parsed_def = _eval_literal_default(params.get("items") or params.get("initial_items", "()"))
tuple_expr = nctx.resolve_input("items", repr(parsed_def) if isinstance(parsed_def, (list, tuple, set)) else format_literal(parsed_def))
if not tuple_expr.strip().startswith(("[", "(", "{")):
cg.discovered_modules.add("ast")
tuple_expr = f"(ast.literal_eval({tuple_expr}) if isinstance({tuple_expr}, str) else {tuple_expr})"
lines.append(f" {nv} = tuple({tuple_expr})")
ovm.update({f"{node_id}:result": nv, f"{node_id}:tuple_out": nv, f"{node_id}:out": nv, f"{node_id}:count": f"len({nv})", node_id: nv})
return True, nv
if raw_type in ("data.set", "set"):
raw_set_def = _eval_literal_default(params.get("initial_items") or params.get("initial_elements", "[]"))
elements_expr = nctx.resolve_input(["set_a", "items", "initial_elements"], repr(raw_set_def) if isinstance(raw_set_def, (list, tuple, set)) else format_literal(raw_set_def))
if not elements_expr.strip().startswith(("[", "(", "{")):
cg.discovered_modules.add("ast")
elements_expr = f"(ast.literal_eval({elements_expr}) if isinstance({elements_expr}, str) else {elements_expr})"
nv = f"set_{safe_id}"
lines.append(f" {nv} = set({elements_expr})")
add_expr = nctx.resolve_input(["add_item", "element"], "None")
if add_expr != "None":
lines.append(f" {nv}.add({add_expr})")
ovm.update({f"{node_id}:result": nv, f"{node_id}:set_out": nv, f"{node_id}:out": nv, f"{node_id}:count": f"len({nv})", f"{node_id}:cardinality": f"len({nv})", node_id: nv})
return True, nv
return False, None
|