Buckets:
| """IPython magics.""" | |
| from __future__ import annotations | |
| import ast | |
| import os | |
| import pathlib | |
| from IPython.core import magic_arguments | |
| from IPython.core.magic import Magics, cell_magic, magics_class | |
| import calkit | |
| from calkit.dvc import run_dvc_command | |
| # This code used to ensure we can import from the project root in stage | |
| # scripts | |
| PROJECT_ROOT_PATH_PY_CODE = """ | |
| import os | |
| import sys | |
| sys.path.insert( | |
| 0, | |
| os.path.abspath( | |
| os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../") | |
| ), | |
| ) | |
| """ | |
| def _parse_string_arg(val: str): | |
| try: | |
| return ast.literal_eval(val) | |
| except (ValueError, SyntaxError): | |
| return val | |
| return val | |
| def _posix_path(path: str): | |
| return pathlib.Path(path).as_posix() | |
| class Calkit(Magics): | |
| def stage(self, line, cell): | |
| """Turn a notebook cell into a DVC pipeline stage. | |
| Note that all dependencies must be declared since the cell will be | |
| first turned into a script and then run as part of the DVC pipeline. | |
| Then variables will be loaded back into the user namespace state by | |
| loading the DVC output. | |
| """ | |
| # Allow command for this line to continue with backslashes | |
| cell_split = cell.split("\n") | |
| line_combined = line.strip().removesuffix("\\") | |
| while line.strip().endswith("\\"): | |
| newline = cell_split.pop(0) | |
| line += newline | |
| line_combined += " " + newline.strip().removesuffix("\\") | |
| cell = "\n".join(cell_split) | |
| args = magic_arguments.parse_argstring(self.stage, line_combined) | |
| # If an output object type is specified, make sure we only have one | |
| # output | |
| if args.out_type: | |
| all_outs = [] | |
| if args.out: | |
| all_outs += args.out | |
| if args.out_path: | |
| all_outs = args.out_path | |
| if len(all_outs) != 1: | |
| raise ValueError( | |
| "Only one output can be defined if declaring as a " | |
| "Calkit object" | |
| ) | |
| # Parse calkit object parameters | |
| out_params = {} | |
| if args.out_title: | |
| out_params["title"] = args.out_title | |
| if args.out_desc: | |
| out_params["description"] = args.out_desc | |
| # Ensure we have required keys | |
| # TODO: Use Pydantic here | |
| if "title" not in out_params: | |
| raise ValueError( | |
| f"Calkit type {args.out_type} requires a title" | |
| ) | |
| # Parse output path | |
| if args.out_path: | |
| out_params["path"] = args.out_path[0] | |
| elif args.out: | |
| out = args.out[0] | |
| out_split = out.split(":") | |
| kws = dict(stage_name=args.name, out_name=out_split[0]) | |
| if len(out_split) > 1: | |
| kws["fmt"] = out_split[1] | |
| out_path = calkit.get_notebook_stage_out_path(**kws) | |
| out_params["path"] = out_path | |
| out_params["stage"] = args.name | |
| # Save in calkit.yaml | |
| ck_info = calkit.load_calkit_info() | |
| objs = ck_info.get(args.out_type + "s", []) | |
| objs = [obj for obj in objs if obj["path"] != out_params["path"]] | |
| objs.append(out_params) | |
| ck_info[args.out_type + "s"] = objs | |
| with open("calkit.yaml", "w") as f: | |
| calkit.ryaml.dump(ck_info, f) | |
| # First, let's write this cell out to a script, ensuring that we | |
| # load the important state at the top | |
| script_txt = "# This script was automatically generated by Calkit\n\n" | |
| script_txt += PROJECT_ROOT_PATH_PY_CODE + "\n\n" | |
| script_txt += "import calkit\n\n" | |
| if args.dep: | |
| for d in args.dep: | |
| dep_split = d.split(":") | |
| stage = dep_split[0] | |
| varname = dep_split[1] | |
| fmt_string = "" | |
| eng_string = "" | |
| if len(dep_split) >= 3: | |
| fmt_string = f", fmt='{dep_split[2]}'" | |
| if len(dep_split) == 4: | |
| eng_string = f", engine='{dep_split[3]}'" | |
| script_txt += ( | |
| f"{varname} = calkit.load_notebook_stage_out(" | |
| f"stage_name='{stage}', out_name='{varname}'" | |
| f"{fmt_string}{eng_string})\n\n" | |
| ) | |
| script_txt += cell | |
| # Add lines that save our output variables to files | |
| if args.out: | |
| for out in args.out: | |
| fmt_string = "" | |
| eng_string = "" | |
| out_split = out.split(":") | |
| outvar = out_split[0] | |
| if len(out_split) > 1: | |
| fmt_string = f", fmt='{out_split[1]}'" | |
| if len(out_split) == 3: | |
| eng_string = f", engine='{out_split[2]}'" | |
| script_txt += ( | |
| f"calkit.save_notebook_stage_out(" | |
| f"{outvar}, stage_name='{args.name}', out_name='{outvar}'" | |
| f"{fmt_string}{eng_string})\n" | |
| ) | |
| # Save the script to a Python file | |
| script_fpath = calkit.get_notebook_stage_script_path(args.name) | |
| script_dir = os.path.dirname(script_fpath) | |
| os.makedirs(script_dir, exist_ok=True) | |
| outs_dir = calkit.get_notebook_stage_out_dir(stage_name=args.name) | |
| os.makedirs(outs_dir, exist_ok=True) | |
| with open(script_fpath, "w") as f: | |
| f.write(script_txt) | |
| # Create a DVC stage that runs the script, defining the appropriate | |
| # dependencies and outputs | |
| # TODO: Insert this into dvc.yaml directly, since DVC reformats | |
| cmd = [ | |
| "stage", | |
| "add", | |
| "-n", | |
| args.name, | |
| "--force", | |
| "-d", | |
| _posix_path(script_fpath), | |
| ] | |
| if not args.verbose: | |
| cmd.append("-q") | |
| if args.dep: | |
| for dep in args.dep: | |
| dep_split = dep.split(":") | |
| stage = dep_split[0] | |
| varname = dep_split[1] | |
| kws = dict(stage_name=stage, out_name=varname) | |
| if len(dep_split) > 2: | |
| kws["fmt"] = dep_split[2] | |
| cmd += [ | |
| "-d", | |
| _posix_path(calkit.get_notebook_stage_out_path(**kws)), | |
| ] | |
| if args.dep_path: | |
| for dep in args.dep_path: | |
| cmd += ["-d", _posix_path(dep)] | |
| if args.out: | |
| for out in args.out: | |
| out_split = out.split(":") | |
| out_name = out_split[0] | |
| kws = dict(stage_name=args.name, out_name=out_name) | |
| if len(out_split) > 1: | |
| kws["fmt"] = out_split[1] | |
| # Figure out DVC caching | |
| if args.out_storage == "dvc": | |
| out_flag = "-o" | |
| else: | |
| out_flag = "--outs-no-cache" | |
| cmd += [ | |
| out_flag, | |
| _posix_path(calkit.get_notebook_stage_out_path(**kws)), | |
| ] | |
| if args.out_path: | |
| for path in args.out_path: | |
| cmd += ["-o", _posix_path(path)] | |
| stage_cmd = f'python "{_posix_path(script_fpath)}"' | |
| if args.env: | |
| xenv = "calkit xenv" | |
| if isinstance(args.env, str): | |
| xenv += f" -n {args.env}" | |
| stage_cmd = xenv + " -- " + stage_cmd | |
| cmd.append(stage_cmd) | |
| result = run_dvc_command(cmd) | |
| if result != 0: | |
| raise RuntimeError(f"Failed to add DVC stage {args.name}") | |
| # Now run the stage | |
| result = run_dvc_command(["repro", args.name]) | |
| if result != 0: | |
| raise RuntimeError(f"DVC repro failed for stage {args.name}") | |
| # Now let's read in and inject the outputs back into the IPython state | |
| if args.out: | |
| for out in args.out: | |
| out_split = out.split(":") | |
| out_name = out_split[0] | |
| kws = dict(stage_name=args.name, out_name=out_name) | |
| if len(out_split) > 1: | |
| kws["fmt"] = out_split[1] | |
| if len(out_split) > 2: | |
| kws["engine"] = out_split[2] | |
| self.shell.user_ns[out_name] = calkit.load_notebook_stage_out( | |
| **kws | |
| ) | |
| # If the last line of the cell has no equals signs, run that command, | |
| # since it's probably meant for display | |
| last_line = cell.strip().split("\n")[-1] | |
| if "=" not in last_line: | |
| self.shell.run_cell(last_line) | |
| def load_ipython_extension(ipython): | |
| """Any module file that define a function named `load_ipython_extension` | |
| can be loaded via `%load_ext module.path` or be configured to be | |
| autoloaded by IPython at startup time. | |
| See https://ipython.readthedocs.io/en/stable/config/custommagics.html | |
| """ | |
| # You can register the class itself without instantiating it | |
| # IPython will call the default constructor on it | |
| ipython.register_magics(Calkit) | |
Xet Storage Details
- Size:
- 12.2 kB
- Xet hash:
- 40cfb73da754cca9c4b65b659e437ed9094d13bec31eafba5a0c9619626adec5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.