Buckets:
| """Functionality for working with notebooks.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING, Any, Literal | |
| if TYPE_CHECKING: | |
| from calkit.models.io import ( | |
| InputsFromStageOutputs, | |
| PathInput, | |
| PathOutput, | |
| ) | |
| def get_executed_notebook_path( | |
| notebook_path: str, | |
| to: Literal["html", "notebook"], | |
| as_posix: bool = True, | |
| parameters: dict[str, Any] | None = None, | |
| ) -> str: | |
| """Return the path of an executed notebook.""" | |
| nb_dir = os.path.dirname(notebook_path) | |
| nb_fname = os.path.basename(notebook_path) | |
| # If we have any parameters, add these to the notebook name | |
| if parameters: | |
| parameters_cleaned = {} | |
| for k, v in parameters.items(): | |
| if isinstance(v, list): | |
| if len(v) == 0: | |
| v = "empty" | |
| elif len(v) < 5: | |
| v = ",".join(map(str, v)) | |
| else: | |
| v = hashlib.md5( | |
| json.dumps(v, sort_keys=True).encode() | |
| ).hexdigest()[:7] | |
| elif isinstance(v, dict): | |
| v = hashlib.md5( | |
| json.dumps(v, sort_keys=True).encode() | |
| ).hexdigest()[:7] | |
| parameters_cleaned[k] = v | |
| params_txt = "-".join( | |
| f"{k}-{v}" for k, v in parameters_cleaned.items() | |
| ) | |
| nb_fname = f"{nb_fname.removesuffix('.ipynb')}-{params_txt}.ipynb" | |
| if to == "html": | |
| fname_out = nb_fname.removesuffix(".ipynb") + ".html" | |
| else: | |
| fname_out = nb_fname | |
| subdirs = {"html": "html", "notebook": "executed"} | |
| p = os.path.join(".calkit", "notebooks", subdirs[to], nb_dir, fname_out) | |
| if as_posix: | |
| p = Path(p).as_posix() | |
| return p | |
| # How much of a file to read when deciding whether it's a marimo notebook. | |
| # marimo writes its header at the top, but a project's own license banner or | |
| # module docstring comes first, so leave room for one. | |
| MARIMO_DETECT_N_BYTES = 8192 | |
| def is_marimo_notebook(content: str) -> bool: | |
| """Determine whether some Python source is a marimo notebook. | |
| A marimo notebook is a Python module that constructs a ``marimo.App``, | |
| which is both what makes it one and what marimo itself writes at the top | |
| of every notebook it generates. Keyed on the constructor call rather | |
| than on the word 'marimo' appearing somewhere, so a script that merely | |
| imports or mentions marimo isn't mistaken for a notebook. | |
| """ | |
| return "marimo.App(" in content | |
| def get_cleaned_notebook_path(path: str, as_posix: bool = True) -> str: | |
| """Return the path of a cleaned notebook.""" | |
| p = os.path.join(".calkit", "notebooks", "cleaned", path) | |
| if as_posix: | |
| p = Path(p).as_posix() | |
| return p | |
| def clean_notebook(nb: dict) -> dict: | |
| """Strip a notebook's outputs, execution counts, and metadata in place. | |
| Cell tags survive, since they drive parameterization; everything else | |
| that isn't source is execution residue. The result is what a notebook | |
| stage depends on, so that re-running a notebook doesn't make its stage | |
| stale while editing a cell does. | |
| """ | |
| for cell in nb.get("cells", []): | |
| if cell.get("cell_type") == "code": | |
| cell["outputs"] = [] | |
| cell["execution_count"] = None | |
| # Clean metadata but keep tags | |
| if "tags" in cell.get("metadata", {}): | |
| cell["metadata"] = {"tags": cell["metadata"]["tags"]} | |
| else: | |
| cell["metadata"] = {} | |
| # Clean out notebook-level metadata | |
| nb["metadata"] = {} | |
| return nb | |
| def clean_notebook_outputs(path: str): | |
| """Clean the outputs of a notebook and put it in the cleaned notebooks | |
| dir. | |
| """ | |
| if os.path.isabs(path): | |
| raise ValueError("Path must be relative") | |
| fpath_out = get_cleaned_notebook_path(path) | |
| folder = os.path.dirname(fpath_out) | |
| os.makedirs(folder, exist_ok=True) | |
| fpath_out = os.path.abspath(fpath_out) | |
| with open(path, "r", encoding="utf-8") as f: | |
| try: | |
| nb = json.load(f) | |
| except json.JSONDecodeError as e: | |
| raise ValueError(f"Error decoding JSON from notebook: {e}") | |
| with open(fpath_out, "w", encoding="utf-8") as f: | |
| json.dump(clean_notebook(nb), f, indent=2) | |
| def clean_all_in_pipeline(ck_info: dict | None = None) -> list[str]: | |
| """Clean all notebooks in the pipeline.""" | |
| if ck_info is None: | |
| import calkit | |
| ck_info = calkit.load_calkit_info() | |
| pipeline = ck_info.get("pipeline", {}) | |
| stages = pipeline.get("stages", {}) | |
| cleaned = [] | |
| for _, stage in stages.items(): | |
| if stage.get("kind") == "jupyter-notebook": | |
| path = stage.get("notebook_path") | |
| if path: | |
| clean_notebook_outputs(path) | |
| cleaned.append(path) | |
| return cleaned | |
| def determine_storage( | |
| notebook_path: str, | |
| size_multiplier: float = 3.0, | |
| ) -> Literal["git", "dvc"]: | |
| if not os.path.exists(notebook_path): | |
| return "dvc" | |
| try: | |
| file_size = os.path.getsize(notebook_path) | |
| except OSError: | |
| return "dvc" | |
| source_bytes = 0 | |
| try: | |
| with open(notebook_path, "r", encoding="utf-8") as f: | |
| nb = json.load(f) | |
| for cell in nb.get("cells", []): | |
| source = cell.get("source", []) | |
| if isinstance(source, list): | |
| source_bytes += len("".join(source).encode("utf-8")) | |
| else: | |
| source_bytes += len(str(source).encode("utf-8")) | |
| except (json.JSONDecodeError, UnicodeDecodeError, OSError): | |
| source_bytes = file_size | |
| estimated_bytes = max(file_size, source_bytes) * size_multiplier | |
| from calkit.core import DVC_SIZE_THRESH_BYTES | |
| return "git" if estimated_bytes <= DVC_SIZE_THRESH_BYTES else "dvc" | |
| def declare_notebook( | |
| path: str, | |
| stage_name: str, | |
| environment_name: str, | |
| inputs: list[str | PathInput | InputsFromStageOutputs] = [], | |
| outputs: list[str | PathOutput] = [], | |
| always_run: bool = False, | |
| title: str | None = None, | |
| description: str | None = None, | |
| html_storage: Literal["git", "dvc"] | None = "dvc", | |
| executed_ipynb_storage: Literal["git", "dvc"] | None = "dvc", | |
| cleaned_ipynb_storage: Literal["git", "dvc"] | None = "git", | |
| ): | |
| """Declare a notebook as part of the current project. | |
| If this function is called as part of a running pipeline, it will fail | |
| if anything has changed about the pipeline declaration, then prompt the | |
| user to rerun. | |
| """ | |
| import calkit | |
| from calkit.models.pipeline import JupyterNotebookStage, Pipeline | |
| # Detect the project root directory so we ensure calkit.yaml lives there | |
| repo = calkit.git.get_repo() | |
| wdir = repo.working_dir | |
| if not os.path.isfile(os.path.join(wdir, path)): | |
| raise FileNotFoundError( | |
| f"Notebook '{path}' does not exist in the project directory" | |
| ) | |
| pipeline_running = os.getenv("CALKIT_PIPELINE_RUNNING", "0") == "1" | |
| ck_info = calkit.load_calkit_info(wdir=str(wdir)) | |
| envs = ck_info.get("environments", {}) | |
| if environment_name not in envs: | |
| raise ValueError( | |
| f"Environment '{environment_name}' does not exist in calkit.yaml" | |
| ) | |
| # TODO: Check that we are running in the correct environment | |
| # This could be tricky depending on what type of environment it is | |
| pipeline_dict = ck_info.get("pipeline", {}) | |
| if "stages" not in pipeline_dict: | |
| pipeline_dict["stages"] = {} | |
| pipeline = Pipeline.model_validate(pipeline_dict) | |
| new_stage = JupyterNotebookStage( | |
| notebook_path=path, | |
| environment=environment_name, | |
| inputs=inputs, | |
| outputs=outputs, | |
| always_run=always_run, | |
| html_storage=html_storage, | |
| executed_ipynb_storage=executed_ipynb_storage, | |
| cleaned_ipynb_storage=cleaned_ipynb_storage, | |
| ) | |
| stages = pipeline.stages | |
| must_be_rerun = False | |
| # Check to see if we're mutating an existing stage | |
| if pipeline_running and stage_name in stages: | |
| # If the pipeline is running, we can't change the stage | |
| existing_stage = stages[stage_name] | |
| if existing_stage != new_stage: | |
| must_be_rerun = True | |
| new_stage_dump = new_stage.model_dump() | |
| new_stage_dict = { | |
| "kind": "jupyter-notebook", | |
| "notebook_path": path, | |
| "environment": environment_name, | |
| } | |
| if inputs: | |
| new_stage_dict["inputs"] = new_stage_dump["inputs"] | |
| if outputs: | |
| new_stage_dict["outputs"] = new_stage_dump["outputs"] | |
| if always_run: | |
| new_stage_dict["always_run"] = always_run # type: ignore | |
| new_stage_dict.update( | |
| { | |
| "html_storage": html_storage, | |
| "executed_ipynb_storage": executed_ipynb_storage, | |
| "cleaned_ipynb_storage": cleaned_ipynb_storage, | |
| } # type: ignore | |
| ) | |
| pipeline_dict["stages"][stage_name] = new_stage_dict | |
| # Update existing notebook if it exists, else append | |
| notebooks = ck_info.get("notebooks", []) | |
| updated = False | |
| for notebook in notebooks: | |
| if notebook.get("path") == path: | |
| notebook.update( | |
| { | |
| "title": title, | |
| "description": description, | |
| "stage": stage_name, | |
| } | |
| ) | |
| updated = True | |
| break | |
| if not updated: | |
| notebooks.append( | |
| { | |
| "path": path, | |
| "title": title, | |
| "description": description, | |
| "stage": stage_name, | |
| } | |
| ) | |
| # Write calkit.yaml | |
| ck_info["pipeline"] = pipeline_dict | |
| ck_info["notebooks"] = notebooks | |
| calkit.save_calkit_info(ck_info, wdir=wdir) | |
| if must_be_rerun: | |
| raise RuntimeError( | |
| f"Notebook stage '{stage_name}' was modified while the pipeline " | |
| "was running; please run the pipeline again" | |
| ) | |
| # TODO: Change to the correct working directory? | |
Xet Storage Details
- Size:
- 10.1 kB
- Xet hash:
- 38d3081e4cea8a539a01acf4676f55c4bf53653056a3c6f46544e64da810563f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.