Buckets:

download
raw
10.1 kB
"""Functionality for working with notebooks."""
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
if TYPE_CHECKING:
from calkit.models.io import (
InputsFromStageOutputs,
PathInput,
PathOutput,
)
def get_executed_notebook_path(
notebook_path: str,
to: Literal["html", "notebook"],
as_posix: bool = True,
parameters: dict[str, Any] | None = None,
) -> str:
"""Return the path of an executed notebook."""
nb_dir = os.path.dirname(notebook_path)
nb_fname = os.path.basename(notebook_path)
# If we have any parameters, add these to the notebook name
if parameters:
parameters_cleaned = {}
for k, v in parameters.items():
if isinstance(v, list):
if len(v) == 0:
v = "empty"
elif len(v) < 5:
v = ",".join(map(str, v))
else:
v = hashlib.md5(
json.dumps(v, sort_keys=True).encode()
).hexdigest()[:7]
elif isinstance(v, dict):
v = hashlib.md5(
json.dumps(v, sort_keys=True).encode()
).hexdigest()[:7]
parameters_cleaned[k] = v
params_txt = "-".join(
f"{k}-{v}" for k, v in parameters_cleaned.items()
)
nb_fname = f"{nb_fname.removesuffix('.ipynb')}-{params_txt}.ipynb"
if to == "html":
fname_out = nb_fname.removesuffix(".ipynb") + ".html"
else:
fname_out = nb_fname
subdirs = {"html": "html", "notebook": "executed"}
p = os.path.join(".calkit", "notebooks", subdirs[to], nb_dir, fname_out)
if as_posix:
p = Path(p).as_posix()
return p
# How much of a file to read when deciding whether it's a marimo notebook.
# marimo writes its header at the top, but a project's own license banner or
# module docstring comes first, so leave room for one.
MARIMO_DETECT_N_BYTES = 8192
def is_marimo_notebook(content: str) -> bool:
"""Determine whether some Python source is a marimo notebook.
A marimo notebook is a Python module that constructs a ``marimo.App``,
which is both what makes it one and what marimo itself writes at the top
of every notebook it generates. Keyed on the constructor call rather
than on the word 'marimo' appearing somewhere, so a script that merely
imports or mentions marimo isn't mistaken for a notebook.
"""
return "marimo.App(" in content
def get_cleaned_notebook_path(path: str, as_posix: bool = True) -> str:
"""Return the path of a cleaned notebook."""
p = os.path.join(".calkit", "notebooks", "cleaned", path)
if as_posix:
p = Path(p).as_posix()
return p
def clean_notebook(nb: dict) -> dict:
"""Strip a notebook's outputs, execution counts, and metadata in place.
Cell tags survive, since they drive parameterization; everything else
that isn't source is execution residue. The result is what a notebook
stage depends on, so that re-running a notebook doesn't make its stage
stale while editing a cell does.
"""
for cell in nb.get("cells", []):
if cell.get("cell_type") == "code":
cell["outputs"] = []
cell["execution_count"] = None
# Clean metadata but keep tags
if "tags" in cell.get("metadata", {}):
cell["metadata"] = {"tags": cell["metadata"]["tags"]}
else:
cell["metadata"] = {}
# Clean out notebook-level metadata
nb["metadata"] = {}
return nb
def clean_notebook_outputs(path: str):
"""Clean the outputs of a notebook and put it in the cleaned notebooks
dir.
"""
if os.path.isabs(path):
raise ValueError("Path must be relative")
fpath_out = get_cleaned_notebook_path(path)
folder = os.path.dirname(fpath_out)
os.makedirs(folder, exist_ok=True)
fpath_out = os.path.abspath(fpath_out)
with open(path, "r", encoding="utf-8") as f:
try:
nb = json.load(f)
except json.JSONDecodeError as e:
raise ValueError(f"Error decoding JSON from notebook: {e}")
with open(fpath_out, "w", encoding="utf-8") as f:
json.dump(clean_notebook(nb), f, indent=2)
def clean_all_in_pipeline(ck_info: dict | None = None) -> list[str]:
"""Clean all notebooks in the pipeline."""
if ck_info is None:
import calkit
ck_info = calkit.load_calkit_info()
pipeline = ck_info.get("pipeline", {})
stages = pipeline.get("stages", {})
cleaned = []
for _, stage in stages.items():
if stage.get("kind") == "jupyter-notebook":
path = stage.get("notebook_path")
if path:
clean_notebook_outputs(path)
cleaned.append(path)
return cleaned
def determine_storage(
notebook_path: str,
size_multiplier: float = 3.0,
) -> Literal["git", "dvc"]:
if not os.path.exists(notebook_path):
return "dvc"
try:
file_size = os.path.getsize(notebook_path)
except OSError:
return "dvc"
source_bytes = 0
try:
with open(notebook_path, "r", encoding="utf-8") as f:
nb = json.load(f)
for cell in nb.get("cells", []):
source = cell.get("source", [])
if isinstance(source, list):
source_bytes += len("".join(source).encode("utf-8"))
else:
source_bytes += len(str(source).encode("utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError, OSError):
source_bytes = file_size
estimated_bytes = max(file_size, source_bytes) * size_multiplier
from calkit.core import DVC_SIZE_THRESH_BYTES
return "git" if estimated_bytes <= DVC_SIZE_THRESH_BYTES else "dvc"
def declare_notebook(
path: str,
stage_name: str,
environment_name: str,
inputs: list[str | PathInput | InputsFromStageOutputs] = [],
outputs: list[str | PathOutput] = [],
always_run: bool = False,
title: str | None = None,
description: str | None = None,
html_storage: Literal["git", "dvc"] | None = "dvc",
executed_ipynb_storage: Literal["git", "dvc"] | None = "dvc",
cleaned_ipynb_storage: Literal["git", "dvc"] | None = "git",
):
"""Declare a notebook as part of the current project.
If this function is called as part of a running pipeline, it will fail
if anything has changed about the pipeline declaration, then prompt the
user to rerun.
"""
import calkit
from calkit.models.pipeline import JupyterNotebookStage, Pipeline
# Detect the project root directory so we ensure calkit.yaml lives there
repo = calkit.git.get_repo()
wdir = repo.working_dir
if not os.path.isfile(os.path.join(wdir, path)):
raise FileNotFoundError(
f"Notebook '{path}' does not exist in the project directory"
)
pipeline_running = os.getenv("CALKIT_PIPELINE_RUNNING", "0") == "1"
ck_info = calkit.load_calkit_info(wdir=str(wdir))
envs = ck_info.get("environments", {})
if environment_name not in envs:
raise ValueError(
f"Environment '{environment_name}' does not exist in calkit.yaml"
)
# TODO: Check that we are running in the correct environment
# This could be tricky depending on what type of environment it is
pipeline_dict = ck_info.get("pipeline", {})
if "stages" not in pipeline_dict:
pipeline_dict["stages"] = {}
pipeline = Pipeline.model_validate(pipeline_dict)
new_stage = JupyterNotebookStage(
notebook_path=path,
environment=environment_name,
inputs=inputs,
outputs=outputs,
always_run=always_run,
html_storage=html_storage,
executed_ipynb_storage=executed_ipynb_storage,
cleaned_ipynb_storage=cleaned_ipynb_storage,
)
stages = pipeline.stages
must_be_rerun = False
# Check to see if we're mutating an existing stage
if pipeline_running and stage_name in stages:
# If the pipeline is running, we can't change the stage
existing_stage = stages[stage_name]
if existing_stage != new_stage:
must_be_rerun = True
new_stage_dump = new_stage.model_dump()
new_stage_dict = {
"kind": "jupyter-notebook",
"notebook_path": path,
"environment": environment_name,
}
if inputs:
new_stage_dict["inputs"] = new_stage_dump["inputs"]
if outputs:
new_stage_dict["outputs"] = new_stage_dump["outputs"]
if always_run:
new_stage_dict["always_run"] = always_run # type: ignore
new_stage_dict.update(
{
"html_storage": html_storage,
"executed_ipynb_storage": executed_ipynb_storage,
"cleaned_ipynb_storage": cleaned_ipynb_storage,
} # type: ignore
)
pipeline_dict["stages"][stage_name] = new_stage_dict
# Update existing notebook if it exists, else append
notebooks = ck_info.get("notebooks", [])
updated = False
for notebook in notebooks:
if notebook.get("path") == path:
notebook.update(
{
"title": title,
"description": description,
"stage": stage_name,
}
)
updated = True
break
if not updated:
notebooks.append(
{
"path": path,
"title": title,
"description": description,
"stage": stage_name,
}
)
# Write calkit.yaml
ck_info["pipeline"] = pipeline_dict
ck_info["notebooks"] = notebooks
calkit.save_calkit_info(ck_info, wdir=wdir)
if must_be_rerun:
raise RuntimeError(
f"Notebook stage '{stage_name}' was modified while the pipeline "
"was running; please run the pipeline again"
)
# TODO: Change to the correct working directory?

Xet Storage Details

Size:
10.1 kB
·
Xet hash:
38d3081e4cea8a539a01acf4676f55c4bf53653056a3c6f46544e64da810563f

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.