Buckets:
"""Pipeline-related functionality."""
import itertools
import os
import re
import warnings
from collections.abc import Callable
from pathlib import Path
from typing import Any
import typer
from pydantic import BaseModel, Field, computed_field, field_validator
import calkit
from calkit.models.iteration import expand_project_parameters
from calkit.models.pipeline import (
InputsFromStageOutputs,
LatexStage,
PathOutput,
Pipeline,
)
def _coerce_bool(value: object) -> bool:
"""Interpret a raw YAML value the same way the Stage model would.
``ck_info`` holds unvalidated YAML, so a quoted ``"false"`` (e.g.
``frozen: "false"`` or ``always_run: "false"``) is a truthy string here;
defer to Pydantic's bool parsing so status filtering matches the compiled
pipeline's behavior.
"""
from pydantic import TypeAdapter, ValidationError
try:
return TypeAdapter(bool).validate_python(value)
except ValidationError:
return bool(value)
def frozen_stage_base_names(
ck_info: dict | None = None, wdir: str | None = None
) -> set[str]:
"""Return the base names of root pipeline stages marked ``frozen``.
Shared by the various status paths so frozen stages are consistently
hidden. Iterated stages use a ``name@param`` key in DVC status, so callers
should compare against the part before ``@``.
"""
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
names: set[str] = set()
stages = ck_info.get("pipeline", {}).get("stages", {})
if isinstance(stages, dict):
for name, cfg in stages.items():
if isinstance(cfg, dict) and _coerce_bool(
cfg.get("frozen", False)
):
names.add(name)
return names
class PipelineStatus(BaseModel):
"""Current status of the project pipeline."""
has_pipeline: bool
environment_checks: dict[str, dict] = Field(default_factory=dict)
cleaned_notebooks: list[str] = Field(default_factory=list)
stale_stages: dict[str, "StaleStage"] = Field(default_factory=dict)
errors: list[str] = Field(default_factory=list)
# Git-ignored files inside directory inputs of stages that are not
# currently stale on that input: {stage_name: {input_path: [file, ...]}}.
# DVC hashes them along with the rest of the directory, but Git doesn't
# carry them to other machines, so the stage will look stale elsewhere,
# e.g., in CI. Inputs currently reported as modified are annotated on the
# StaleStage instead, so the two never overlap.
ignored_files_in_inputs: dict[str, dict[str, list[str]]] = Field(
default_factory=dict
)
@field_validator("stale_stages", mode="before")
@classmethod
def _coerce_stale_stages(cls, value):
if not isinstance(value, dict):
return {}
coerced = {}
for stage_name, stage_value in value.items():
if isinstance(stage_value, StaleStage):
coerced[stage_name] = stage_value
else:
coerced[stage_name] = StaleStage.from_status_data(stage_value)
return coerced
@computed_field
@property
def stale_stage_names(self) -> list[str]:
# Pure always-run stages aren't truly stale; surface them via
# always_run_stage_names instead.
return [
name
for name, stage in self.stale_stages.items()
if not stage.always_run
or stage.modified_command
or stage.modified_inputs
or stage.modified_outputs
or stage.stale_outputs
]
@computed_field
@property
def always_run_stage_names(self) -> list[str]:
# Only list stages whose sole change indicator is always_run; stages
# that also have real changes are reported under stale_stage_names.
# Subproject stages are excluded: their always-changed status is a
# delegation detail, not a user-meaningful always-run stage of the
# parent project.
return [
name
for name, stage in self.stale_stages.items()
if stage.always_run
and not stage.is_subproject
and not stage.modified_command
and not stage.modified_inputs
and not stage.modified_outputs
and not stage.stale_outputs
]
@computed_field
@property
def failed_environment_checks(self) -> list[str]:
return sorted(
[
env_name
for env_name, result in self.environment_checks.items()
if not result.get("success", False)
]
)
@computed_field
@property
def is_stale(self) -> bool:
# A stage whose only listed change is "always_run" is not actually
# stale — it just always re-executes by design. Treat the pipeline
# as stale only if some stage has real change indicators.
for stage in self.stale_stages.values():
if not stage.always_run:
return True
if (
stage.modified_command
or stage.modified_inputs
or stage.modified_outputs
or stage.stale_outputs
):
return True
return False
class StaleStage(BaseModel):
"""Structured status information for a stale pipeline stage."""
raw_status: list | dict | str | None = None
stale_outputs: list[str] = Field(default_factory=list)
modified_inputs: list[str] = Field(default_factory=list)
modified_outputs: list[str] = Field(default_factory=list)
modified_command: bool = False
always_run: bool = False
# Git-ignored files inside modified directory inputs, keyed by the entry
# in ``modified_inputs``. DVC hashes a directory as a whole, so an ignored
# file can change the hash without anything showing in Git status; these
# are the usual suspects when a stage looks stale for no visible reason.
ignored_files_in_inputs: dict[str, list[str]] = Field(default_factory=dict)
# True for stages that originate from a subproject (either an individual
# "{sp}:{stage}" stage or a kept "{sp} (subproject)" wrapper). Subproject
# wrapper stages are marked always-changed purely as a delegation
# mechanism, so they should not be advertised as always-run stages of the
# parent project.
is_subproject: bool = False
@staticmethod
def _as_path_list(paths: object) -> list[str]:
if isinstance(paths, str):
return [paths]
if isinstance(paths, list):
return [str(path) for path in paths]
return []
@classmethod
def _collect_paths(cls, change_group: object) -> list[str]:
paths = []
if isinstance(change_group, list):
for item in change_group:
paths.extend(cls._collect_paths(item))
return list(dict.fromkeys(paths))
if not isinstance(change_group, dict):
return []
for key, values in change_group.items():
# DVC may encode path->change_type, e.g., {"foo.txt": "modified"}
if isinstance(values, str) and values in {
"modified",
"new",
"deleted",
}:
paths.append(str(key))
else:
paths.extend(cls._as_path_list(values))
return list(dict.fromkeys(paths))
@classmethod
def _collect_output_paths(
cls, change_group: object
) -> tuple[list[str], list[str]]:
all_paths = []
changed_paths = []
changed_statuses = {"modified"}
known_statuses = changed_statuses | {
"new",
"deleted",
"not in cache",
"always changed",
}
if isinstance(change_group, list):
for item in change_group:
item_all, item_changed = cls._collect_output_paths(item)
all_paths.extend(item_all)
changed_paths.extend(item_changed)
return list(dict.fromkeys(all_paths)), list(
dict.fromkeys(changed_paths)
)
if not isinstance(change_group, dict):
return [], []
for key, values in change_group.items():
if isinstance(values, str) and values in known_statuses:
path = str(key)
all_paths.append(path)
if values in changed_statuses:
changed_paths.append(path)
continue
if isinstance(values, list):
listed_paths = [str(path) for path in values]
if key in known_statuses:
all_paths.extend(listed_paths)
if key in changed_statuses:
changed_paths.extend(listed_paths)
else:
all_paths.extend(listed_paths)
continue
item_all, item_changed = cls._collect_output_paths(values)
all_paths.extend(item_all)
changed_paths.extend(item_changed)
return list(dict.fromkeys(all_paths)), list(
dict.fromkeys(changed_paths)
)
@classmethod
def from_status_data(
cls,
status_data: list | dict | str,
configured_outputs: list[str] | None = None,
path_prefix: str | None = None,
is_subproject: bool = False,
declared_always_run: bool = False,
) -> "StaleStage":
modified_inputs = []
output_paths = []
modified_outputs = []
status_blocks = []
modified_command = False
marker_always_changed = False
if isinstance(status_data, dict):
status_blocks = [status_data]
elif isinstance(status_data, list):
status_blocks = [
item for item in status_data if isinstance(item, dict)
]
# DVC may return plain markers like ["changed command"] or
# ["always changed"], possibly alongside other change blocks.
if "changed command" in status_data:
modified_command = True
if "always changed" in status_data:
marker_always_changed = True
elif isinstance(status_data, str):
modified_command = status_data == "changed command"
marker_always_changed = status_data == "always changed"
# DVC emits the "always changed" marker for two unrelated reasons: a
# stage compiled with always_changed: true (the user set always_run),
# or a stage that simply has no outputs to track. Only the former is a
# genuine always-run stage. A never-run, output-less stage must be
# reported as stale rather than silently hidden as always-run.
# Subproject wrapper stages carry the marker purely as a delegation
# mechanism, so they continue to count as always-run.
always_run = marker_always_changed and (
declared_always_run or is_subproject
)
for block in status_blocks:
if "changed command" in block:
changed_command_value = block.get("changed command")
# Any explicit non-false value indicates the stage command
# changed and outputs should be considered stale.
modified_command = modified_command or (
changed_command_value is None
or changed_command_value is True
or bool(changed_command_value)
)
modified_inputs.extend(
cls._collect_paths(
block.get("changed deps", block.get("deps", {}))
)
)
out_paths, changed_out_paths = cls._collect_output_paths(
block.get("changed outs", block.get("outs", {}))
)
output_paths.extend(out_paths)
modified_outputs.extend(changed_out_paths)
modified_inputs = list(dict.fromkeys(modified_inputs))
output_paths = list(dict.fromkeys(output_paths))
modified_outputs = list(dict.fromkeys(modified_outputs))
# Prefix all DVC-reported paths with the subproject directory when
# viewing status from the parent repo, so all paths are consistently
# parent-relative. Use normpath to collapse any "../" segments that
# arise from cross-subproject deps (e.g. sp + "../shared.txt").
if path_prefix:
def _prefix(p: str) -> str:
return Path(
os.path.normpath(os.path.join(path_prefix, p))
).as_posix()
modified_inputs = [_prefix(p) for p in modified_inputs]
output_paths = [_prefix(p) for p in output_paths]
modified_outputs = [_prefix(p) for p in modified_outputs]
configured_outputs = [str(path) for path in (configured_outputs or [])]
stale_outputs = []
if modified_inputs or modified_command:
stale_outputs.extend(configured_outputs)
stale_outputs.extend(
[path for path in output_paths if path not in modified_outputs]
)
# Fallback: if DVC flagged the stage but we couldn't attribute the
# change to specific paths, mark all configured outputs as stale.
# Skip the fallback for pure "always_run" stages — they always
# re-execute by design, not because outputs are out of date.
if (
not stale_outputs
and not modified_outputs
and configured_outputs
and not always_run
):
stale_outputs.extend(configured_outputs)
stale_outputs = list(dict.fromkeys(stale_outputs))
return cls(
raw_status=status_data,
stale_outputs=stale_outputs,
modified_inputs=modified_inputs,
modified_outputs=modified_outputs,
modified_command=modified_command,
always_run=always_run,
is_subproject=is_subproject,
)
def stages_are_similar(stage1: dict, stage2: dict) -> bool:
"""Check if two stage configurations are fundamentally the same.
Compares stage kind and key parameters to determine if stages represent
the same operation.
Parameters
----------
stage1 : dict
First stage configuration.
stage2 : dict
Second stage configuration.
Returns
-------
bool
True if stages are similar, False otherwise.
"""
# Different kind means different stage
if stage1.get("kind") != stage2.get("kind"):
return False
kind = stage1.get("kind")
# For script stages, check script path and args
if kind in [
"python-script",
"julia-script",
"matlab-script",
"shell-script",
]:
if stage1.get("script_path") != stage2.get("script_path"):
return False
if stage1.get("args", []) != stage2.get("args", []):
return False
# For notebook stages
elif kind == "jupyter-notebook":
if stage1.get("notebook_path") != stage2.get("notebook_path"):
return False
# For latex and quarto
elif kind in ["latex", "quarto"]:
if stage1.get("target_path") != stage2.get("target_path"):
return False
# For command stages, check the command
elif kind in [
"command",
"shell-command",
"matlab-command",
"julia-command",
]:
if stage1.get("command") != stage2.get("command"):
return False
return True
def _expand_matrix(input_dict: dict[str, list]) -> list[dict]:
"""Restructure a dictionary with list values into a list of dictionaries,
where each dictionary represents a permutation of the input dictionary's
values.
"""
keys = list(input_dict.keys())
values = list(input_dict.values())
# Create all combinations of values using itertools.product
combinations = itertools.product(*values)
# Create a list of dictionaries
list_of_dicts = []
for combination in combinations:
list_of_dicts.append(dict(zip(keys, combination)))
# After expanding the matrix, flatten any nested dictionaries in the result
# This handles cases where list-of-lists iterations produce dictionaries as
# values, by concatenating parent and child keys (e.g., "parent.child")
# so all permutations are represented as flat dictionaries
final_list = []
for item in list_of_dicts:
keys = list(item.keys())
vals = list(item.values())
vd = {}
for key, val in zip(keys, vals):
if isinstance(val, dict):
for k, v in val.items():
vd[f"{key}.{k}"] = v
else:
vd[key] = val
final_list.append(vd)
return final_list
def _paths_overlap(a: str, b: str) -> bool:
"""Return True if path `a` and path `b` overlap in the file tree.
Two paths overlap if they are equal or one is an ancestor directory of the
other, e.g. ``data/`` overlaps with ``data/file.csv``.
"""
a = a.rstrip("/")
b = b.rstrip("/")
if a == b:
return True
return b.startswith(a + "/") or a.startswith(b + "/")
def _expand_dep_excluding_subprojects(
dep: str,
isolated_sp_paths: list[str],
root: str | None = None,
) -> list[str]:
"""Expand a directory dep to avoid traversing into isolated subprojects.
DVC cannot collect a directory dep that contains a nested DVC repo (i.e.
an isolated subproject identified by a ``.dvc/`` directory). This function
walks ``dep``, listing its contents and skipping any isolated subproject
directory, so the returned paths collectively cover the same files without
crossing into the subprojects.
``root`` is the directory relative to which ``dep`` and ``isolated_sp_paths``
are expressed (defaults to cwd).
"""
root_path = Path(root) if root else Path(".")
dep_path = root_path / dep
# Resolve once for reliable path comparison
dep_resolved = dep_path.resolve()
sp_set = {(root_path / sp).resolve() for sp in isolated_sp_paths}
nested = [
sp
for sp in sp_set
if sp != dep_resolved and sp.is_relative_to(dep_resolved)
]
if not nested or not dep_path.is_dir():
return [dep]
result: list[str] = []
def _walk(dir_path: Path) -> None:
try:
children = sorted(dir_path.iterdir())
except PermissionError:
return
for child in children:
child_resolved = child.resolve()
if child_resolved in sp_set:
# Skip the isolated subproject entirely
continue
if child.is_dir() and any(
sp.is_relative_to(child_resolved) for sp in nested
):
# This directory is an ancestor of an isolated SP; descend
_walk(child)
else:
rel = child.relative_to(root_path)
suffix = "/" if child.is_dir() else ""
result.append(rel.as_posix() + suffix)
_walk(dep_path)
return result if result else [dep]
def collapse_dvc_stages(
stages: dict,
cmd: str | None = None,
wdir: str | None = None,
desc: str | None = None,
) -> dict:
"""Collapse a dict of DVC stages into a single wrapper stage.
Collects all deps and outs from ``stages``, expands matrix templates, and
returns a complete DVC stage dict. ``cmd``, ``wdir``, and ``desc`` are
optional; when provided they are included in the returned dict.
Rules applied:
- A dep that tree-overlaps any out is dropped (e.g. a folder dep that
contains output files would cause a DVC conflict).
- An out that tree-overlaps any remaining dep is also dropped (covers
exact-match aliasing and folder/child relationships).
- Outputs are wrapped with ``{cache: false, persist: true}`` so the parent
doesn't double-cache files already managed by the subproject's DVC.
"""
all_outs: set[str] = set()
all_deps: set[str] = set()
for stage_cfg in stages.values():
matrix = stage_cfg.get("matrix")
replacements = _expand_matrix(matrix) if matrix else [{}]
for out in stage_cfg.get("outs", []):
raw = out if isinstance(out, str) else list(out.keys())[0]
for r in replacements:
expanded = raw
for var, val in r.items():
expanded = expanded.replace(f"${{item.{var}}}", str(val))
all_outs.add(Path(expanded).as_posix())
for dep in stage_cfg.get("deps", []):
if not isinstance(dep, str):
continue
for r in replacements:
expanded = dep
for var, val in r.items():
expanded = expanded.replace(f"${{item.{var}}}", str(val))
all_deps.add(Path(expanded).as_posix())
external_deps = all_deps - all_outs
outs_raw = sorted(all_outs)
# Track whether any dep is dropped because it is a strict folder ancestor
# of an output (not an exact match — those are already removed by the set
# subtraction above). When this happens the wrapper cannot fully express
# the input boundary, so we mark it always_changed.
# A dep is a strict ancestor of an out when the out path starts with the
# dep path followed by "/". The reverse (dep inside an output folder) is
# just a normal internal path drop and does not require always_changed.
has_ancestor_dep_drop = any(
o.startswith(d.rstrip("/") + "/")
for d in external_deps
for o in outs_raw
)
deps = sorted(
d
for d in external_deps
if not any(_paths_overlap(d, o) for o in outs_raw)
)
dep_set = set(deps)
outs = [
{o: {"cache": False, "persist": True}}
for o in outs_raw
if not any(_paths_overlap(o, d) for d in dep_set)
]
stage: dict = {}
if cmd is not None:
stage["cmd"] = cmd
if wdir is not None:
stage["wdir"] = wdir
if has_ancestor_dep_drop:
stage["always_changed"] = True
stage["deps"] = deps
stage["outs"] = outs
if desc is not None:
stage["desc"] = desc
return stage
def _status_target_matches(
target: str,
display_name: str,
bare_name: str,
subproject: str | None,
stale_stage: StaleStage,
) -> bool:
"""Return True if a status ``target`` selects this stage.
Because the full DVC status is now computed and filtered locally (rather
than asking DVC to filter by target), this is where each ``target`` passed
to :func:`get_status` is resolved against a single stage. A ``target`` may
be a stage name in any of calkit's forms (``stage``, ``dvc.yaml:stage``,
``subproject:stage``, or a whole ``subproject``) or a repo path, which is
matched against the stage's stale/modified inputs and outputs.
Iterated/matrix stages use a ``name@param`` key in DVC status, so the base
name (the part before ``@``) is matched too — targeting the base name
selects every iteration, consistent with how ``calkit run`` expands matrix
targets.
"""
base_name = bare_name.split("@")[0]
# Stage-name matching is done against the raw target so it covers all of
# calkit's stage-target forms as well as DVC's native ``dvc.yaml:stage``
# and ``<subproject>/dvc.yaml:stage`` forms (the latter is what
# translate_run_targets emits for inline subprojects).
stage_aliases = {display_name, bare_name, base_name}
if subproject is not None:
subproject_name = Path(subproject).name
if target in {subproject, subproject_name}:
return display_name.startswith(
f"{subproject}:"
) or display_name == (f"{subproject} (subproject)")
stage_aliases |= {
f"{subproject}:{bare_name}",
f"{subproject_name}:{bare_name}",
f"{subproject}:{base_name}",
f"{subproject_name}:{base_name}",
f"{subproject}/dvc.yaml:{bare_name}",
f"{subproject}/dvc.yaml:{base_name}",
}
else:
stage_aliases |= {
f"dvc.yaml:{bare_name}",
f"dvc.yaml:{base_name}",
}
if target in stage_aliases:
return True
# Otherwise treat the target as a repo path and match it against the
# stage's stale/modified inputs and outputs. A target is path-like if it
# exists on disk or contains a separator; this keeps root-level paths
# without separators (e.g. ``data`` or ``out.csv``) selectable, which a
# separator-only heuristic would miss. A target that looks like a stage
# selector (contains ``:``) but matched no stage above is not a path.
if ":" in target and not os.path.exists(target):
return False
is_pathlike = os.path.exists(target) or "/" in target or "\\" in target
if not is_pathlike:
return False
norm_target = Path(target).as_posix().rstrip("/")
candidate_paths = (
stale_stage.stale_outputs
+ stale_stage.modified_inputs
+ stale_stage.modified_outputs
)
return any(_paths_overlap(norm_target, p) for p in candidate_paths)
def get_status(
ck_info: dict | None = None,
targets: list[str] | None = None,
wdir: str | None = None,
check_environments: bool = True,
clean_notebooks: bool = True,
compile_to_dvc: bool = True,
force_env_check: bool = False,
) -> PipelineStatus:
"""Get pipeline status after optional prep checks.
This can compile the Calkit pipeline to DVC, clean notebook outputs,
check pipeline environments, then query DVC for out-of-date stages.
"""
import calkit.environments
import calkit.markdown
prev_cwd = os.getcwd()
if wdir is not None:
os.chdir(wdir)
try:
if ck_info is None:
ck_info = calkit.load_calkit_info()
has_pipeline = bool(ck_info.get("pipeline", {}).get("stages", {}))
has_pipeline = has_pipeline or os.path.isfile("dvc.yaml")
has_subprojects = bool(ck_info.get("subprojects"))
result: dict[str, Any] = {
"has_pipeline": has_pipeline or has_subprojects,
"environment_checks": {},
"cleaned_notebooks": [],
"stale_stages": {},
"errors": [],
}
if not has_pipeline and not has_subprojects:
return PipelineStatus.model_validate(result)
# Environment checking and status filtering both read stages and
# environments straight off this dict, so markdown stages have to
# become ordinary ones before either runs.
try:
ck_info = calkit.markdown.expand_ck_info(ck_info).ck_info
except Exception as e:
result["errors"].append(
f"Failed to read markdown stages: {e.__class__.__name__}: {e}"
)
return PipelineStatus.model_validate(result)
if check_environments:
try:
env_checks = calkit.environments.check_all_in_pipeline(
ck_info=ck_info,
targets=targets,
force=force_env_check,
)
result["environment_checks"] = env_checks
except Exception as e:
result["errors"].append(
"Failed to check pipeline environments: "
f"{e.__class__.__name__}: {e}"
)
return PipelineStatus.model_validate(result)
failed_env_checks = [
env_name
for env_name, info in env_checks.items()
if not info.get("success", False)
]
if failed_env_checks:
return PipelineStatus.model_validate(result)
if compile_to_dvc and (
ck_info.get("pipeline", {}).get("stages", {})
or ck_info.get("subprojects")
):
try:
to_dvc(ck_info=ck_info, write=True, manage_gitignore=False)
except Exception as e:
result["errors"].append(
f"Failed to compile pipeline: {e.__class__.__name__}: {e}"
)
return PipelineStatus.model_validate(result)
if clean_notebooks and ck_info.get("pipeline", {}).get("stages", {}):
try:
cleaned = calkit.notebooks.clean_all_in_pipeline(
ck_info=ck_info
)
result["cleaned_notebooks"] = cleaned
except Exception as e:
result["errors"].append(
"Failed to clean notebooks in pipeline: "
f"{e.__class__.__name__}: {e}"
)
return PipelineStatus.model_validate(result)
from dvc.lock import LockError
try:
dvc_repo = calkit.dvc.get_dvc_repo()
# calkit.dvc.core installs a filter on dvc.repo.status that drops
# the noisy frozen-stage warning, so the call here stays quiet.
# Ask DVC for the complete stale set, then filter locally.
# DVC's own target filtering can miss stale propagation from
# isolated subprojects, which makes targeted status disagree with
# the full-project status view.
raw_status = dvc_repo.status()
raw_status = calkit.dvc.status_as_posix(raw_status)
except LockError:
# Another DVC process is holding the repo lock---most often a
# long-running `dvc pull`/`push` or a concurrent `calkit run`. We
# genuinely cannot compute status until it releases, so state what
# is running plainly rather than surfacing an alarming error (e.g.
# to the VS Code extension's status poller). The holder PID is
# reliable here because we just failed to acquire the lock. We've
# already chdir'd into wdir above, so read the lock from the cwd
# rather than re-joining the (possibly relative) wdir.
holder = calkit.dvc.get_dvc_lock_holder()
if holder is not None and holder.get("cmd"):
result["errors"].append(
f"A DVC process is in progress (PID {holder['pid']}: "
f"{holder['cmd']}). Status will be available once it "
"finishes."
)
else:
result["errors"].append(
"Another DVC process is holding the lock (a run or pull "
"may be in progress). Status will be available once it "
"finishes."
)
return PipelineStatus.model_validate(result)
except Exception as e:
result["errors"].append(
"Failed to get pipeline status from DVC: "
f"{e.__class__.__name__}: {e}"
)
return PipelineStatus.model_validate(result)
# Isolated subprojects appear in the parent status as wrapper stages
# named after the subproject directory (e.g., "dvc.yaml:sub1"). When
# a wrapper stage is stale, replace it with the individual stale stages
# from the subproject's own DVC so the user sees {sp}:stage_name detail.
sp_by_stage_name: dict[str, str] = {}
isolated_sp_paths: set[str] = set()
for sp_cfg in ck_info.get("subprojects", []):
if not isinstance(sp_cfg, dict) or not sp_cfg.get("path"):
continue
sp = Path(sp_cfg["path"]).as_posix()
if os.path.isdir(os.path.join(sp, ".dvc")):
sp_by_stage_name[Path(sp).name] = sp
isolated_sp_paths.add(sp)
# Root-level stage keys have the form "stage_name" (no dvc.yaml: prefix).
stale_wrapper_keys = [
k
for k in list(raw_status.keys())
if (k.split("dvc.yaml:", 1)[-1] if "dvc.yaml:" in k else k)
in sp_by_stage_name
]
for wrapper_key in stale_wrapper_keys:
bare_wrapper = (
wrapper_key.split("dvc.yaml:", 1)[1]
if "dvc.yaml:" in wrapper_key
else wrapper_key
)
sp = sp_by_stage_name[bare_wrapper]
try:
sp_dvc_repo = calkit.dvc.get_dvc_repo(sp)
sp_raw_status = sp_dvc_repo.status()
sp_raw_status = calkit.dvc.status_as_posix(sp_raw_status)
except Exception:
sp_raw_status = {}
if sp_raw_status:
# Sub-project has its own stale stages: replace the wrapper key
# with individual stage keys so the user sees {sp}:stage_name.
del raw_status[wrapper_key]
sp_is_isolated = os.path.isdir(os.path.join(sp, ".dvc"))
for k, v in sp_raw_status.items():
# For isolated subprojects the DVC repo is rooted at sp, so
# keys are sp-relative and need the sp/ prefix.
# For inline subprojects the DVC repo is the parent root, so
# keys already contain the sp path (e.g. "sub1/dvc.yaml:…").
if sp_is_isolated:
if "dvc.yaml:" in k:
_, bare = k.split("dvc.yaml:", 1)
raw_status[f"{sp}/dvc.yaml:{bare}"] = v
else:
raw_status[f"{sp}/dvc.yaml:{k}"] = v
else:
raw_status[k] = v
# else: keep the wrapper key — sub-project is internally up-to-date
# but the parent's dvc.lock needs refreshing.
# DVC status keys have the form:
# "dvc.yaml:stage_name" root pipeline
# "sub1/dvc.yaml:stage_name" subproject pipeline
# Parse each key into (display_name, bare_stage_name, subproject_path).
# Using a display_name of "sub1/stage_name" for subproject stages avoids
# collisions when a subproject stage shares a name with a root stage.
raw_stale_stages: dict[str, tuple[str, str | None, list]] = {}
for k, v in raw_status.items():
# Skip DVC-tracked files (.dvc entries) — they are not pipeline
# stages. Stages marked ["always changed"] are kept so the user
# sees them in the pipeline status output.
if k.endswith(".dvc"):
continue
if "dvc.yaml:" in k:
prefix, bare_name = k.split("dvc.yaml:", 1)
subproject = prefix.rstrip("/") if prefix else None
else:
bare_name = k
subproject = None
if subproject:
display_name = f"{subproject}:{bare_name}"
elif bare_name in sp_by_stage_name:
# Wrapper stage kept in place (sub-project internally up-to-
# date but parent dvc.lock needs refreshing).
display_name = f"{sp_by_stage_name[bare_name]} (subproject)"
else:
display_name = bare_name
raw_stale_stages[display_name] = (bare_name, subproject, v)
root_stages_config = ck_info.get("pipeline", {}).get("stages", {})
# Lazily load subproject ck_info for configured_outputs lookup
sp_stages_config: dict[str, dict] = {}
for subproject in ck_info.get("subprojects", []):
if not isinstance(subproject, dict) or not subproject.get("path"):
continue
sp = Path(subproject["path"]).as_posix()
try:
sp_ck = calkit.load_calkit_info(wdir=sp)
sp_stages_config[sp] = sp_ck.get("pipeline", {}).get(
"stages", {}
)
except Exception:
pass
# Build stage ordering from root dvc.yaml; subproject stages sort after.
dvc_yaml_stages: list[str] = []
dvc_yaml_stage_defs: dict[str, dict] = {}
if os.path.isfile("dvc.yaml"):
try:
with open("dvc.yaml") as f:
dvc_yaml = calkit.ryaml.load(f)
dvc_yaml_stage_defs = (dvc_yaml or {}).get("stages", {})
dvc_yaml_stages = list(dvc_yaml_stage_defs.keys())
except Exception:
pass
ordered_stale_stages = {}
dvc_yaml_stage_order = {
name: i for i, name in enumerate(dvc_yaml_stages)
}
def _stage_sort_key(display_name: str) -> tuple[int, int]:
# Subproject stages always sort after root stages (bucket 1 vs 0).
bare, sp, _ = raw_stale_stages[display_name]
if sp is None:
base = bare.split("@")[0]
return (
0,
dvc_yaml_stage_order.get(base, len(dvc_yaml_stages)),
)
return (1, 0)
for display_name in sorted(
raw_stale_stages.keys(), key=_stage_sort_key
):
bare_name, subproject, status_data = raw_stale_stages[display_name]
# Frozen stages are intentionally pinned: never report them as
# stale (and they are never reproduced). Iterated stages have a
# "name@param" bare name, so look up config by the base name too.
base_name = bare_name.split("@")[0]
if subproject is None:
frozen_cfg = root_stages_config.get(
bare_name, root_stages_config.get(base_name, {})
)
else:
sp_cfg = sp_stages_config.get(subproject, {})
frozen_cfg = sp_cfg.get(bare_name, sp_cfg.get(base_name, {}))
if isinstance(frozen_cfg, dict) and _coerce_bool(
frozen_cfg.get("frozen", False)
):
continue
if subproject is None:
stage_cfg = root_stages_config.get(bare_name, {})
else:
stage_cfg = sp_stages_config.get(subproject, {}).get(
bare_name, {}
)
# DVC reports subproject stage paths relative to the repo root
# (e.g., "sub1/out.txt"), but calkit.yaml stores them relative to
# the subproject dir ("out.txt"). Prefix with the subproject path
# so configured_outputs matches DVC's reported paths.
raw_outputs = [
output.get("path", str(output))
if isinstance(output, dict)
else str(output)
for output in stage_cfg.get("outputs", [])
]
if subproject:
# Use posix separators so these match DVC's reported paths
# (which are always forward-slash, repo-root-relative). Using
# the OS-native separator here would, on Windows, produce a
# backslash variant that escapes path dedup and shows up as a
# duplicate stale output alongside DVC's posix path.
configured_outputs = [
(Path(subproject) / p).as_posix() for p in raw_outputs
]
else:
configured_outputs = raw_outputs
# For isolated subprojects, DVC reports paths relative to the
# subproject dir; prefix them so all paths are parent-relative.
# A stage is subproject-originating if it carries a subproject
# path ("{sp}:{stage}") or is a kept wrapper for a subproject
# ("{sp} (subproject)", whose bare name is a known subproject).
is_subproject = (
subproject is not None or bare_name in sp_by_stage_name
)
ordered_stale_stages[display_name] = StaleStage.from_status_data(
status_data=status_data,
configured_outputs=configured_outputs,
path_prefix=subproject
if subproject in isolated_sp_paths
else None,
is_subproject=is_subproject,
declared_always_run=_coerce_bool(stage_cfg.get("always_run"))
if isinstance(stage_cfg, dict)
else False,
)
if targets:
ordered_stale_stages = {
display_name: stale_stage
for display_name, stale_stage in ordered_stale_stages.items()
if any(
_status_target_matches(
target,
display_name,
raw_stale_stages[display_name][0],
raw_stale_stages[display_name][1],
stale_stage,
)
for target in targets
)
}
# Git-ignored files inside directory inputs. DVC hashes a directory
# input as a whole without consulting .gitignore, so a stray ignored
# file (scratch data, an editor backup) changes the hash yet shows up
# in neither Git status nor DVC's file-level status. The stage then
# looks stale for no visible reason, and since Git doesn't carry the
# file to other machines, a fresh checkout such as CI hashes the
# directory differently and the stage flips between stale and
# up-to-date depending on where it's run (calkit#1036). Scan the
# compiled dvc.yaml deps rather than calkit.yaml inputs: stage kinds
# add deps of their own (a notebook's include_paths, say) and dvc.yaml
# is exactly what DVC hashes. Only directories are inspected, so file
# deps like cleaned notebooks or environment lock files are never
# considered. Files DVC tracks (e.g., an upstream stage's outputs) are
# ignored by Git on purpose and are excluded, as is anything matched
# by .dvcignore, which DVC skips when hashing.
stale_by_base: dict[str, list[StaleStage]] = {}
for display_name, stale_stage in ordered_stale_stages.items():
bare_name, subproject, _ = raw_stale_stages[display_name]
# Only root stages are scanned; iterated stages are keyed
# "name@param" in DVC status, so match on the base name
if subproject is None:
stale_by_base.setdefault(bare_name.split("@")[0], []).append(
stale_stage
)
ignored_files_in_inputs: dict[str, dict[str, list[str]]] = {}
# Directories are shared between stages, so list each one only once.
# The Git repo and DVC index are only resolved once a directory
# actually needs listing, so a pipeline without directory inputs
# never pays for them.
ignored_files_by_dir: dict[str, list[str]] = {}
git_repo = None
git_root = ""
dvc_out_paths: list[str] = []
for name, stage_def in dvc_yaml_stage_defs.items():
if not isinstance(stage_def, dict):
continue
for dep in stage_def.get("deps", []) or []:
if not isinstance(dep, str):
continue
# Deps are relative to the stage's wdir, if it has one
dep_path = Path(
os.path.normpath(
os.path.join(stage_def.get("wdir", ""), dep)
)
).as_posix()
if not os.path.isdir(dep_path):
continue
if targets and not any(
target in {name, f"dvc.yaml:{name}"}
or _paths_overlap(Path(target).as_posix(), dep_path)
for target in targets
):
continue
if dep_path not in ignored_files_by_dir:
if git_repo is None:
git_repo = calkit.git.get_repo()
git_root = str(git_repo.working_tree_dir)
dvc_out_paths = [
Path(os.path.relpath(out.fs_path)).as_posix()
for out in dvc_repo.index.outs
]
# Git prints paths relative to the repo root, which may be
# above the project, so make them project-relative; -z
# keeps unusual file names from being quoted
listing = git_repo.git.ls_files(
"-z",
"--others",
"--ignored",
"--exclude-standard",
"--",
os.path.abspath(dep_path),
)
files: list[str] = []
for ignored_path in listing.split("\0"):
if not ignored_path:
continue
ignored_path = Path(
os.path.relpath(
os.path.join(git_root, ignored_path)
)
).as_posix()
if any(
ignored_path == out
or ignored_path.startswith(out + "/")
for out in dvc_out_paths
):
continue
if dvc_repo.dvcignore.is_ignored_file(
os.path.abspath(ignored_path)
):
continue
files.append(ignored_path)
ignored_files_by_dir[dep_path] = files
files = ignored_files_by_dir[dep_path]
if not files:
continue
# When DVC reports the directory (or, if the dep was expanded
# around a subproject, a path inside it) as modified, annotate
# that input on the stale stage; otherwise it goes in the
# status-wide list as something that will bite elsewhere
annotated = False
for stale_stage in stale_by_base.get(name, []):
for modified_input in stale_stage.modified_inputs:
inside = [
p
for p in files
if p == modified_input
or p.startswith(modified_input.rstrip("/") + "/")
]
if inside:
stale_stage.ignored_files_in_inputs[
modified_input
] = inside
annotated = True
if not annotated:
ignored_files_in_inputs.setdefault(name, {})[dep_path] = (
files
)
result["stale_stages"] = ordered_stale_stages
return PipelineStatus(
has_pipeline=result["has_pipeline"],
environment_checks=result["environment_checks"],
cleaned_notebooks=result["cleaned_notebooks"],
stale_stages=result["stale_stages"],
errors=result["errors"],
ignored_files_in_inputs=ignored_files_in_inputs,
)
finally:
if wdir is not None:
os.chdir(prev_cwd)
def get_stage_for_output(path: str, ck_info: dict) -> str | None:
"""Return the name of the pipeline stage that produces ``path``, if any."""
target = Path(path).as_posix()
stages = ck_info.get("pipeline", {}).get("stages", {})
if not isinstance(stages, dict):
return None
for name, stage in stages.items():
if not isinstance(stage, dict):
continue
for out in stage.get("outputs", []):
out_path = out.get("path") if isinstance(out, dict) else out
if out_path and Path(out_path).as_posix() == target:
return name
return None
def get_output_storage_map(
ck_info: dict | None = None,
wdir: str | None = None,
) -> dict[str, str]:
"""Get a map of pipeline output paths to their explicitly-set storage.
Only outputs with an explicitly-set ``storage`` key in ``calkit.yaml``
are included so that default-DVC outputs still go through auto-detection.
Parameters
----------
ck_info : dict | None
Calkit project info dict. Loaded from ``calkit.yaml`` if not provided.
wdir : str | None
Working directory. Defaults to the current working directory.
Returns
-------
dict[str, str]
Mapping of posix file path to storage type, e.g.
``{"figures/plot.png": "git", "data/archive": "dvc-zip"}``.
Plain string outputs (no explicit ``storage`` key) are not included.
"""
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
pipeline = ck_info.get("pipeline", {})
if not pipeline:
return {}
stages = pipeline.get("stages", {})
result: dict[str, str] = {}
for stage in stages.values():
if not isinstance(stage, dict):
continue
for out in stage.get("outputs", []):
if isinstance(out, dict) and "path" in out and "storage" in out:
result[Path(out["path"]).as_posix()] = out["storage"]
return result
def _dump_yaml_if_changed(data, fpath: str, verbose: bool = False) -> bool:
"""Write ``data`` to ``fpath`` as YAML only if it changes the file.
``calkit status`` recompiles the pipeline to DVC on every call, and the VS
Code extension polls status continuously. Blindly rewriting an identical
file causes two problems: (1) bumping ``dvc.yaml``'s mtime wakes the
extension's file watcher, which triggers yet another status poll---an
endless loop that keeps re-taking DVC's lock; and (2) the serializer formats
slightly differently across platforms and ruamel versions, so the file
churns in Git for no real change. Comparison is on the parsed structure, so
trivial reformatting is not treated as a change. DVC keys stage staleness on
``dvc.lock`` (not on ``dvc.yaml``'s mtime), so skipping a no-op write is
safe. Returns whether the file was written.
"""
if os.path.isfile(fpath):
try:
with open(fpath) as f:
if (calkit.ryaml.load(f) or {}) == data:
return False
except Exception:
# Unreadable or invalid existing YAML: fall through and overwrite.
pass
if verbose:
typer.echo(f"Writing to {os.path.basename(fpath)}")
with open(fpath, "w") as f:
calkit.ryaml.dump(data, f)
return True
def _parse_latexmkrc_var(text: str, var: str) -> str | None:
"""Extract a literal ``$<var> = '...'`` assignment from latexmkrc text.
Returns the assigned string for a simple literal assignment (single- or
double-quoted), or ``None`` if absent or computed. Anything non-literal
is intentionally left for the user rather than guessed at.
"""
match = re.search(
r"^\s*\$" + re.escape(var) + r"""\s*=\s*['"]([^'"]*)['"]""",
text,
flags=re.MULTILINE,
)
if match is None:
return None
return match.group(1)
def _warn_on_latexmkrc_out_dir_mismatch(
stage: LatexStage, wdir: str | None = None
) -> None:
"""Warn at compile time if latexmkrc ``$out_dir`` disagrees with the
stage's ``output_dir``.
The tracked PDF output path is derived from ``output_dir``, so a latexmkrc
that redirects the PDF without a matching ``output_dir`` would leave the
real PDF untracked. Because ``calkit latex build`` runs latexmk with
``-cd``, the latexmkrc's ``$out_dir`` is relative to the LaTeX source
directory, whereas ``output_dir`` (like all Calkit path fields) is relative
to the stage's working directory; we resolve the former into that same
frame before comparing. We only read the file when a latexmkrc is
referenced (rare), so there is no cost on the common path.
"""
if stage.latexmkrc_path is None:
return
# Stage path fields are relative to the stage's wdir, which in turn is
# relative to the (sub)project being compiled.
base = os.path.join(wdir or ".", stage.wdir) if stage.wdir else wdir or "."
try:
with open(os.path.join(base, stage.latexmkrc_path)) as f:
text = f.read()
except OSError:
return
rc_out_dir = _parse_latexmkrc_var(text, "out_dir")
if rc_out_dir is None:
# Absent or computed (non-literal): nothing reliable to compare.
return
source_dir = os.path.dirname(stage.target_path)
# Resolve the source-relative $out_dir into the stage's path frame (the
# same frame as output_dir). An empty or "." $out_dir means "next to the
# source", which is what output_dir=None also means. Comparison uses native
# os.path; the resolved value is converted to POSIX only for display.
rc_resolved = os.path.normpath(os.path.join(source_dir or ".", rc_out_dir))
next_to_source = os.path.normpath(source_dir or ".")
if stage.output_dir is None:
declared = next_to_source
else:
declared = os.path.normpath(stage.output_dir)
if rc_resolved == declared:
return
rc_posix = Path(rc_resolved).as_posix()
if rc_resolved == next_to_source:
fix = "remove output_dir (the PDF is written next to the source)"
else:
fix = f"set output_dir: {rc_posix}"
warnings.warn(
f"LaTeX stage '{stage.name}': latexmkrc '{stage.latexmkrc_path}' "
f"sets $out_dir={rc_out_dir!r} (relative to the source, i.e. "
f"{rc_posix!r} in the stage's path frame), but the stage's output_dir "
f"is {stage.output_dir!r}. The tracked PDF output path is derived from "
"output_dir, so these should agree or the compiled PDF may not be "
f"tracked. {fix}.",
stacklevel=2,
)
def _ensure_calkit_gitattributes(wdir: str | None = None) -> bool:
"""Pin the line endings of Calkit's own generated files to LF.
Several of them are committed *and* hashed by DVC as stage
dependencies -- an environment's lock file most of all. Git rewrites
text files to CRLF on checkout when ``core.autocrlf`` is on, which is
the default for Git for Windows, and DVC hashes the working tree, so a
Windows collaborator's checkout disagrees with everyone else's
``dvc.lock`` and the stage reads stale forever. Writing the file with
LF is not enough: the rewrite happens on the way out of Git, after we
are done.
DVC recommends the same mechanism for cross-platform work, but applied
to everything (``* text=auto eol=lf``, plus ``core.autocrlf false``);
see its guide to running on Windows. This rule is deliberately
narrower, since normalizing a project's own files is the project's
call and doing it unasked would renormalize files nobody meant to
touch. The two agree in direction, so a project that follows DVC's
advice as well loses nothing.
``.gitattributes`` outranks ``core.autocrlf``, so this holds however
the user has Git configured.
Scoped to ``.calkit`` --- everything Calkit generates, and nothing a
project writes for itself. Deliberately not narrower than that: the
files here that are both committed and hashed are spread across
``env-locks`` and ``envs``, in ``.json``, ``.yml`` and ``.txt``, and
an allowlist of those would silently stop covering the next one
added. Ignored files inside ``.calkit`` pick up the rule too, which
costs nothing, since Git never rewrites what it isn't tracking.
Locks a project keeps outside ``.calkit`` --- ``uv.lock``,
``pixi.lock``, a ``.python-version`` beside an environment spec ---
are not covered. They are written by their own tools, and normalizing
a project's own files is the project's call; DVC's guide to running on
Windows covers them.
"""
return _write_managed_gitignore_block(
os.path.join(wdir, ".gitattributes") if wdir else ".gitattributes",
marker="calkit generated files",
lines=["/.calkit/** text eol=lf"],
)
def _ensure_latex_aux_gitignore(
stage: LatexStage, wdir: str | None = None
) -> bool:
"""Ensure a managed ``.gitignore`` block ignores LaTeX aux files.
The block lives in the LaTeX source directory's ``.gitignore`` rather than
inside the build/output directory, which may not exist at compile time and
can be wiped by a clean build (``latexmk -C``). The globs are unanchored,
so aux files are caught wherever latexmk writes them relative to the
source -- next to it, or in an ``aux``/output subdirectory via
``$aux_dir``/``$out_dir``/``output_dir``. ``*.pdf`` is never ignored so the
compiled output stays tracked.
Returns True if ``.gitignore`` was modified.
"""
aux_globs = [
"*.aux",
"*.bbl",
"*.bcf",
"*.blg",
"*.fdb_latexmk",
"*.fls",
"*.lof",
"*.lot",
"*.nav",
"*.out",
"*.run.xml",
"*.snm",
"*.synctex.gz",
"*.toc",
"*.vrb",
]
# Stage path fields are relative to the stage's wdir, which in turn is
# relative to the (sub)project being compiled.
base = os.path.join(wdir or ".", stage.wdir) if stage.wdir else wdir or "."
source_dir = os.path.dirname(stage.target_path)
gitignore_path = os.path.join(base, source_dir, ".gitignore")
return _write_managed_gitignore_block(
gitignore_path, marker="calkit latex aux files", lines=aux_globs
)
def sync_markdown(
ck_info: dict | None = None, wdir: str | None = None
) -> "calkit.markdown.MarkdownExpansion":
"""Extract what the project's Markdown files declare, and persist envs.
Run as preprocessing, before environments are checked, because an
environment declared in Markdown has to be in ``calkit.yaml`` before
anything tries to create or enter it.
"""
import calkit.markdown
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
markdown = calkit.markdown.expand_ck_info(ck_info, wdir=wdir)
if markdown.environments:
_write_markdown_environments(markdown, wdir=wdir)
return markdown
def _write_markdown_environments(
markdown: "calkit.markdown.MarkdownExpansion", wdir: str | None = None
) -> bool:
"""Persist Markdown-declared environments into ``calkit.yaml``.
A stage's command runs ``calkit xenv -n <env>`` as a subprocess, which
reads environments back off disk, so an environment that lives only in
memory can't be entered. Writing it here keeps every existing
environment code path working unchanged.
The Markdown stays authoritative: these entries are rewritten from it
on every compile, and each carries a ``description`` saying so.
Returns True if ``calkit.yaml`` was modified.
"""
import calkit.markdown
ck_yaml_path = os.path.join(wdir, "calkit.yaml") if wdir else "calkit.yaml"
if not os.path.isfile(ck_yaml_path):
return False
with open(ck_yaml_path) as f:
data = calkit.ryaml.load(f) or {}
created_envs_key = (
"environments" not in data or data["environments"] is None
)
if created_envs_key:
data["environments"] = {}
envs = data["environments"]
changed = False
for env_name, env in markdown.environments.items():
if envs.get(env_name) == env:
continue
envs[env_name] = env
changed = True
# An environment a file declared on an earlier compile and no longer
# does---renamed, or removed---would otherwise stay behind for good.
# The generated description is what marks an entry as one of these,
# so a user's own environments are never touched.
for env_name in list(envs):
if env_name in markdown.environments:
continue
source = calkit.markdown.env_source_path(envs[env_name])
if source is not None and source in markdown.markdown_paths:
del envs[env_name]
changed = True
if changed:
# Separate a newly created 'environments' section from what comes
# before it, the way the rest of the file is written
if created_envs_key and hasattr(
data, "yaml_set_comment_before_after_key"
):
try:
data.yaml_set_comment_before_after_key(
"environments", before="\n"
)
except Exception:
pass
_dump_yaml_if_changed(data, ck_yaml_path)
return changed
def _write_managed_gitignore_block(
gitignore_path: str, marker: str, lines: list[str]
) -> bool:
"""Create or update a marker-delimited block in a ``.gitignore`` file.
The block is replaced in place if it already exists (so it stays free of
duplicate/stale entries across recompiles) and appended otherwise.
Returns True if the file was modified.
"""
begin = f"# >>> {marker} (managed by calkit) >>>"
end = f"# <<< {marker} <<<"
block = "\n".join([begin] + lines + [end])
existing = ""
if os.path.isfile(gitignore_path):
with open(gitignore_path) as f:
existing = f.read()
block_re = re.compile(
re.escape(begin) + r".*?" + re.escape(end), flags=re.DOTALL
)
if block_re.search(existing):
updated = block_re.sub(lambda _: block, existing)
if updated == existing:
return False
with open(gitignore_path, "w") as f:
f.write(updated)
return True
parent = os.path.dirname(gitignore_path)
if parent:
os.makedirs(parent, exist_ok=True)
separator = (
"" if not existing else ("\n" if existing.endswith("\n") else "\n\n")
)
with open(gitignore_path, "a") as f:
f.write(separator + block + "\n")
return True
def _ref_resolver(
wdir: str | None, paths: list[str] | None = None
) -> Callable[[str], str] | None:
"""Return something that turns a Git revision into its commit hash.
Stages whose inputs are revisions rather than files put the resolved
hash in their command, so DVC can see when a branch has moved. None
when there's no repo to ask, in which case those stages fall back to
running every time.
"""
try:
repo = calkit.git.get_repo(wdir)
except Exception:
return None
def resolve(ref: str) -> str:
sha = calkit.git.resolve_ref(repo, ref)
if sha is None:
# A revision that isn't there at all compiles as written; the
# command reports it properly when the stage runs
return ref
# The commit this revision last changed the document in describes
# the same document as the tip does, and doesn't move when
# something else is committed
return calkit.git.last_change(repo, sha, paths or []) or sha
return resolve
def _stage_ref_resolver(
wdir: str | None, stage
) -> Callable[[str], str] | None:
"""A resolver that pins revisions by what this stage reads."""
paths = [p for p in stage.dvc_deps if not p.startswith(".calkit/")]
return _ref_resolver(wdir, paths)
def to_dvc(
ck_info: dict | None = None,
wdir: str | None = None,
write: bool = False,
verbose: bool = False,
manage_gitignore: bool = True,
) -> dict:
"""Compile a Calkit pipeline to a DVC pipeline.
If a project has subprojects, their dvc.yaml files are compiled (and
written if write=True) recursively. DVC's root repro auto-discovers all
dvc.yaml files in the tree, so no synthetic parent stages are needed.
Returns a dictionary of DVC stages for the `stages` key of a dvc.yaml.
Returns ``{}`` when the project has no pipeline and no subprojects (e.g.
a subproject that is a library with no defined pipeline stages).
"""
import calkit.dvc.zip
import calkit.markdown
from calkit.environments import get_env_input_paths, get_env_lock_fpath
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
if "pipeline" not in ck_info and "subprojects" not in ck_info:
return {}
# Replace markdown stages with the stages and environments their blocks
# declare, before anything else reads this, so env locks, scheduler
# options and iteration need no knowledge of Markdown.
markdown = calkit.markdown.expand_ck_info(ck_info, wdir=wdir)
ck_info = markdown.ck_info
if write and markdown.environments:
_write_markdown_environments(markdown, wdir=wdir)
# Everything Markdown derives is rewritten on every compile, so it
# stays out of Git: the extracted scripts, and the spec file and
# virtualenv of any environment the Markdown declares. One managed
# block covers them all, which avoids a git check-ignore subprocess
# per path.
if write and manage_gitignore and markdown.script_paths:
# Specs and locks are committed---that is how a Calkit project
# records its environments---and so is the tooling's own bootstrap
# (renv's activate.R and .Rprofile, which renv expects in version
# control). What can't be committed is the installed environment
# itself: it is large and holds absolute paths from the machine
# that built it. renv ignores its own library via renv/.gitignore;
# a virtualenv has nothing equivalent, so name it here.
def _env_dir(path: str) -> str:
# An environment described by a spec file in the project
# directory has no directory of its own, and its virtualenv
# sits beside that file
return Path(os.path.dirname(path) or ".").as_posix()
env_dirs = set()
for env in markdown.environments.values():
env_dirs.add(_env_dir(env["path"]))
# An environment the Markdown only references can live under
# .calkit/envs too---that is where xr puts one it builds from the
# code's imports---and its virtualenv needs ignoring just the same.
for env in (ck_info.get("environments") or {}).values():
if not isinstance(env, dict) or not env.get("path"):
continue
env_dir = _env_dir(env["path"])
if env_dir.startswith(".calkit/envs/"):
env_dirs.add(env_dir)
ignore_lines = ["/.calkit/markdown/"] + [
"/.venv/" if env_dir == "." else f"/{env_dir}/.venv/"
for env_dir in sorted(env_dirs)
]
_write_managed_gitignore_block(
os.path.join(wdir or ".", ".gitignore"),
marker="calkit markdown derived files",
lines=ignore_lines,
)
# Compile subproject pipelines recursively.
# For isolated subprojects (those with their own .dvc/ directory), DVC
# won't cross the .dvc/ boundary during --all-pipelines discovery, so we
# generate a single wrapper stage per isolated subproject in the parent
# dvc.yaml. The wrapper's deps/outs capture the I/O boundary so DVC can
# order execution correctly across subprojects.
# For inline subprojects (no .dvc/ dir), DVC discovers them automatically
# via --all-pipelines so no wrapper stage is needed.
wrapper_stages: dict[str, dict] = {}
isolated_sp_paths: list[str] = []
for subproject in ck_info.get("subprojects", []):
if not isinstance(subproject, dict) or not subproject.get("path"):
raise ValueError("Subprojects must have a 'path' defined")
sp = Path(subproject["path"]).as_posix()
if not os.path.isdir(sp):
raise NotADirectoryError(f"Subproject path '{sp}' does not exist")
sp_is_isolated = os.path.isdir(os.path.join(sp, ".dvc"))
# Always compile the subproject's dvc.yaml recursively.
# Inline subprojects need their dvc.yaml written so DVC discovers them
# via --all-pipelines; isolated subprojects get a wrapper stage instead.
sp_dvc_stages = to_dvc(
wdir=sp,
write=write,
verbose=verbose,
manage_gitignore=manage_gitignore,
)
if not sp_is_isolated:
continue
isolated_sp_paths.append(sp)
if not sp_dvc_stages:
warnings.warn(
f"Subproject '{sp}' has no pipeline stages defined; "
"no wrapper stage will be created.",
stacklevel=2,
)
continue
sp_stage_name = Path(sp).name
wrapper = collapse_dvc_stages(
sp_dvc_stages,
cmd="calkit dvc repro",
wdir=sp,
desc=(
f"Automatically generated wrapper for subproject '{sp}'. "
"Changes made here will be overwritten."
),
)
# calkit.yaml is the source of truth for the subproject pipeline;
# tracking it ensures the wrapper goes stale when the pipeline
# definition changes.
if "calkit.yaml" not in wrapper["deps"]:
wrapper["deps"] = sorted(wrapper["deps"] + ["calkit.yaml"])
wrapper_stages[sp_stage_name] = wrapper
if "pipeline" not in ck_info:
if write and wrapper_stages:
dvc_yaml_fpath = (
os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml"
)
existing = {}
if os.path.isfile(dvc_yaml_fpath):
with open(dvc_yaml_fpath) as f:
existing = calkit.ryaml.load(f) or {}
existing.setdefault("stages", {}).update(wrapper_stages)
_dump_yaml_if_changed(existing, dvc_yaml_fpath)
return wrapper_stages
# Detect stages with the old ``slurm:`` field from the raw data, since
# validation renames it to ``scheduler`` and the validated stages no
# longer say which spelling they were written with.
raw_stages = ck_info.get("pipeline", {}).get("stages", {})
_pre_validate_slurm_stages = [
sname for sname, sdata in raw_stages.items() if "slurm" in sdata
]
try:
pipeline = Pipeline.model_validate(ck_info["pipeline"])
except Exception as e:
raise ValueError(f"Pipeline is not defined properly: {e}")
conflicts = set(wrapper_stages) & set(pipeline.stages)
if conflicts:
raise ValueError(
f"Subproject wrapper stage name(s) conflict with parent pipeline "
f"stage name(s): {sorted(conflicts)}. Rename the subproject "
f"directory or the conflicting stage(s)."
)
# Seed parent stages with wrapper stages so the isolated subproject
# stages appear in dvc.yaml before the parent's own stages.
dvc_stages: dict = dict(wrapper_stages)
# Read existing dvc.yaml now so we can clean up stale .gitignore entries
# when stage outputs are renamed or removed
if write:
dvc_yaml_path = os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml"
if os.path.isfile(dvc_yaml_path):
with open(dvc_yaml_path) as f:
existing_dvc_yaml = calkit.ryaml.load(f)
else:
existing_dvc_yaml = {}
if existing_dvc_yaml is None:
existing_dvc_yaml = {}
existing_dvc_stages = existing_dvc_yaml.get("stages", {})
else:
existing_dvc_stages = {}
# First, gather up any env lock paths we might need for DVC deps.
# Include both inner and outer environments so SLURM/PBS env locks are
# picked up when used as the outer env in a composite environment.
used_envs: set[str] = set()
for stage in pipeline.stages.values():
used_envs.add(stage.inner_environment)
used_envs.add(stage.outer_environment)
env_lock_fpaths: dict[str, list[str]] = {}
environments = ck_info.get("environments", {})
for env_name, env in environments.items():
if env_name not in used_envs:
continue
# An env that runs setup commands can name the files they read---a
# setup script it sources, most often. Those are inputs to every
# stage using the env, in the same way its lock file is: editing
# how a build is set up changes what the build produces. Declared
# rather than parsed out of the commands, since a shell command
# doesn't reliably say which of its words is a path. Only the kinds
# with a ``default_setup``: a Docker env's ``inputs`` are files
# copied into the image, and their checksums already ride along in
# its lock file, which stages depend on.
if env.get("kind") in ("system", "slurm", "pbs"):
for env_input in get_env_input_paths(env, env_name):
env_lock_fpaths.setdefault(env_name, []).append(
Path(env_input).as_posix()
)
lock_fpath = get_env_lock_fpath(
env=env, env_name=env_name, as_posix=True, for_dvc=True, wdir=wdir
)
if lock_fpath is None:
continue
# get_env_lock_fpath prefixes wdir for most env types so the returned
# path is parent-relative (e.g., "sub1/.calkit/env-locks/main").
# When writing a subproject dvc.yaml, deps must be relative to that
# file's directory, so strip the wdir prefix if present.
if wdir:
try:
lock_fpath = Path(lock_fpath).relative_to(wdir).as_posix()
except ValueError:
pass
env_lock_fpaths.setdefault(env_name, []).append(lock_fpath)
# A uv environment's interpreter comes from a .python-version next
# to its spec, not from the lock, which records only a
# requires-python floor. Without it as a dependency, changing the
# pin would leave every stage in this environment looking current.
if env.get("kind") == "uv":
spec_dir = os.path.dirname(env.get("path", ""))
pv_path = Path(
os.path.join(spec_dir, ".python-version")
).as_posix()
pv_full = os.path.join(wdir, pv_path) if wdir else pv_path
if os.path.isfile(pv_full):
env_lock_fpaths[env_name].append(pv_path)
# An environment that installs the project itself---by pointing at
# the project's own spec file, or by naming the project root as a
# path dependency---is a pointer to the working tree, so the lock
# says nothing about the code it resolves to: the source has to be
# a dependency in its own right or editing the package would leave
# its stages looking current.
language = calkit.markdown.env_local_source_language(env, wdir=wdir)
if language is not None:
for source_path in calkit.markdown.local_package_source_paths(
language, wdir=wdir
):
if source_path not in env_lock_fpaths[env_name]:
env_lock_fpaths[env_name].append(source_path)
project_params = expand_project_parameters(ck_info.get("parameters", {}))
# Convert legacy sbatch stages to shell-script + scheduler and rename
# old `slurm:` stage fields to `scheduler:`, updating calkit.yaml
# in-place when write=True so the file stays in sync.
converted_stages = pipeline.convert_sbatch_stages()
slurm_field_stages = [
sname
for sname in _pre_validate_slurm_stages
if sname not in converted_stages
]
if write and (converted_stages or slurm_field_stages):
for sname in converted_stages:
typer.echo(
f"Converted legacy 'sbatch' stage '{sname}' to 'shell-script' "
"with 'scheduler' options in calkit.yaml"
)
for sname in slurm_field_stages:
typer.echo(
f"Renamed 'slurm' field to 'scheduler' in stage '{sname}' "
"in calkit.yaml"
)
ck_yaml_path = (
os.path.join(wdir, "calkit.yaml") if wdir else "calkit.yaml"
)
with open(ck_yaml_path) as _f:
ck_yaml_data = calkit.ryaml.load(_f) or {}
for sname, sdata in converted_stages.items():
ck_yaml_data["pipeline"]["stages"][sname] = sdata
for sname in slurm_field_stages:
stage_data = ck_yaml_data["pipeline"]["stages"][sname]
stage_data["scheduler"] = stage_data.pop("slurm")
_dump_yaml_if_changed(ck_yaml_data, ck_yaml_path)
# Validate and initialize scheduler options (SLURM/PBS) for stages
# whose outer environment is a job scheduler. Env defaults are applied
# at job-submission time, not here.
pipeline.set_stage_scheduler_options(environments=environments)
# Tell procedure stages where their procedure is written down, which
# only calkit.yaml knows
pipeline.resolve_procedure_paths(procedures=ck_info.get("procedures", {}))
if write:
# The resolved setup commands each stage runs, written beside the
# pipeline so the compiled command can name a path instead of
# carrying shell-quoted text, which no single quoting survives on
# both cmd.exe and a POSIX shell. Written after the merge above,
# which is what works out the list.
setup_files = [
path
for stage in pipeline.stages.values()
if (path := stage.write_setup_file(wdir=wdir)) is not None
]
# Not committed: unlike an import's lock, this is derived from
# calkit.yaml and rewritten by every compile, which is to say by
# every 'calkit run' and 'calkit status' -- the same reason a
# cleaned notebook isn't committed either.
if manage_gitignore and setup_files:
_write_managed_gitignore_block(
os.path.join(wdir, ".gitignore") if wdir else ".gitignore",
marker="calkit stage setup",
lines=["/.calkit/stage-setup/"],
)
# Only when there is a lock file to protect. A project with no
# environment that locks under .calkit has nothing whose hash a
# line-ending rewrite could change, and shouldn't get a
# .gitattributes it never needed.
if manage_gitignore and any(
Path(p).as_posix().startswith(".calkit/env-locks/")
for paths in env_lock_fpaths.values()
for p in paths
):
_ensure_calkit_gitattributes(wdir=wdir)
# Ensure environment lock files are set as stage inputs if necessary
pipeline.ensure_env_lock_paths_are_inputs(env_lock_fpaths=env_lock_fpaths)
# Now convert Calkit stages into DVC stages
for stage_name, stage in pipeline.stages.items():
# If this stage is a Jupyter notebook stage, we need to update its
# parameters if any reference project-level parameters
if stage.kind == "jupyter-notebook":
stage.update_parameters(params=project_params)
dvc_stage = stage.to_dvc()
# Check if this stage iterates, which means we should create a matrix
# stage
if stage.iterate_over is not None:
# Process a list of iterations into a DVC matrix stage
# Initialize a DVC matrix
dvc_matrix = {}
# Initialize a dict for doing string formatting on the DVC stage
format_dict = {}
for n, iteration in enumerate(stage.iterate_over):
arg_name = iteration.arg_name
exp_vals = iteration.expand_values(params=project_params)
if isinstance(arg_name, list):
dvc_arg_name = f"_arg{n}"
for arg_name_i in arg_name:
item_string = f"${{item.{dvc_arg_name}.{arg_name_i}}}"
format_dict[arg_name_i] = item_string
else:
dvc_arg_name = arg_name
format_dict[arg_name] = f"${{item.{arg_name}}}"
dvc_matrix[dvc_arg_name] = exp_vals
try:
cmd = dvc_stage["cmd"]
cmd = cmd.format(**format_dict)
dvc_stage["cmd"] = cmd
except Exception as e:
raise ValueError(
(
f"Failed to format cmd '{cmd}': "
f"{e.__class__.__name__}: {e}"
)
)
formatted_deps = []
formatted_outs = []
for dep in dvc_stage.get("deps", []):
try:
formatted_deps.append(dep.format(**format_dict))
except Exception as e:
raise ValueError(
(
f"Failed to format dep '{dep}' with "
f"'{format_dict}': "
f"{e.__class__.__name__}: {e}"
)
)
for out in dvc_stage.get("outs", []):
try:
if isinstance(out, dict):
formatted_outs.append(
{
str(list(out.keys())[0]).format(
**format_dict
): dict(list(out.values())[0])
}
)
else:
formatted_outs.append(out.format(**format_dict))
except Exception as e:
raise ValueError(
(
f"Failed to format out '{out}' with "
f"'{format_dict}': "
f"{e.__class__.__name__}: {e}"
)
)
dvc_stage["deps"] = formatted_deps
dvc_stage["outs"] = formatted_outs
dvc_stage["matrix"] = dvc_matrix
# Expand any directory deps that contain isolated subprojects so DVC
# doesn't try to traverse into their .dvcignore files
if isolated_sp_paths:
expanded_deps = []
for dep in dvc_stage.get("deps", []):
expanded_deps.extend(
_expand_dep_excluding_subprojects(
dep, isolated_sp_paths, root=wdir
)
)
dvc_stage["deps"] = expanded_deps
# Add a description to the DVC stage
desc = (
f"Automatically generated from the '{stage_name}' stage "
"in calkit.yaml. Changes made here will be overwritten."
)
dvc_stage["desc"] = desc
dvc_stages[stage_name] = dvc_stage
# A Calkit stage can compile into more than one DVC stage when the
# work has genuinely different inputs
# Per stage, since which commits count depends on what the stage
# reads. Never cached across calls: a resolver is bound to one
# repo, and a process can compile pipelines in several.
for extra_name, extra_stage in stage.extra_dvc_stages(
resolve_ref=_stage_ref_resolver(wdir, stage)
).items():
# Raised rather than worked around with a suffix: a generated
# name is addressable (calkit run <name>) and is the stage's
# identity in dvc.lock, so a name that shifted when the
# pipeline changed would cause spurious re-runs and orphaned
# lock entries. Renaming the conflicting stage is a one-word
# fix; an unpredictable name isn't.
if extra_name in pipeline.stages or extra_name in dvc_stages:
raise ValueError(
f"Stage '{stage_name}' generates a stage named "
f"'{extra_name}', which already exists; rename the "
"conflicting stage"
)
dvc_stages[extra_name] = extra_stage
# Check for any outputs that should be ignored/unignored.
# Skipped when manage_gitignore=False (e.g., status checks) to avoid
# spawning a git subprocess per output (~100 ms each).
if write and manage_gitignore:
repo = calkit.git.get_repo(wdir)
# Ensure we catch any Jupyter Notebook outputs
outputs = stage.outputs.copy()
if stage.kind == "jupyter-notebook":
outputs += stage.notebook_outputs
elif stage.kind == "marimo-html-wasm":
outputs += stage.app_outputs
sched_log = stage.scheduler_log_output
if sched_log is not None:
outputs.append(sched_log)
# Build the set of current DVC output paths so we can detect stale
# .gitignore entries from the previous version of the stage,
# including synthesized outputs like LaTeX PDFs
current_out_paths = set(calkit.dvc.out_paths_from_stage(dvc_stage))
# If this stage already existed, un-ignore any outputs that have
# been renamed or removed so .gitignore does not accumulate stale
# entries (e.g., after a capitalization change in the path)
old_stage = existing_dvc_stages.get(stage_name, {})
for old_path in calkit.dvc.out_paths_from_stage(old_stage):
if old_path not in current_out_paths:
calkit.git.ensure_path_is_not_ignored(repo, path=old_path)
# Deal with any gitignore changes necessary
for out in outputs:
if isinstance(out, PathOutput) and out.storage is None:
calkit.git.ensure_path_is_ignored(repo, path=out.path)
elif isinstance(out, PathOutput) and out.storage == "git":
calkit.git.ensure_path_is_not_ignored(repo, path=out.path)
if out.path.endswith(".ipynb"):
# A notebook stage's storage is declared here, in the
# pipeline, so a clean filter installed in the clone
# (nbstripout) must not overrule it by stripping the
# outputs back out on the way into Git. That would
# commit bytes that don't match what DVC hashed, with
# nothing showing as modified locally.
calkit.git.ensure_path_is_not_filtered(
repo, path=out.path
)
elif isinstance(out, PathOutput) and out.storage == "dvc-zip":
calkit.git.ensure_path_is_ignored(repo, path=out.path)
calkit.dvc.zip.add(out.path, is_stage_output=True)
# For LaTeX stages, warn on a latexmkrc/output_dir mismatch and keep
# the generated aux files out of Git via a managed .gitignore block.
# Globs (not per-file git check-ignore) keep this cheap for large
# pipelines.
if write and isinstance(stage, LatexStage):
_warn_on_latexmkrc_out_dir_mismatch(stage, wdir=wdir)
if manage_gitignore:
_ensure_latex_aux_gitignore(stage, wdir=wdir)
# Now process any inputs from stage outputs
for stage_name, stage in pipeline.stages.items():
for i in stage.inputs:
if isinstance(i, InputsFromStageOutputs):
dvc_outs = dvc_stages[i.from_stage_outputs]["outs"]
for out in dvc_outs:
if out not in dvc_stages[stage_name]["deps"]:
# Handle cases where outs are from a matrix,
# in which case this output could become a list of
# outputs
if isinstance(out, dict):
out = list(out.keys())[0]
if "${item." in out:
extra_outs = []
dvc_matrix = dvc_stages[i.from_stage_outputs][
"matrix"
]
replacements = _expand_matrix(dvc_matrix)
for r in replacements:
out_i = out
for var_name, var_val in r.items():
out_i = out_i.replace(
f"${{item.{var_name}}}",
str(var_val),
)
extra_outs.append(out_i)
for out_i in extra_outs:
if out_i not in dvc_stages[stage_name]["deps"]:
dvc_stages[stage_name]["deps"].append(
out_i
)
else:
dvc_stages[stage_name]["deps"].append(out)
if write:
dvc_yaml = existing_dvc_yaml
existing_stages = existing_dvc_stages
for stage_name, stage in existing_stages.items():
# Preserve any existing stage not already in the new compilation
# and not auto-generated (auto-generated stages are re-emitted
# from scratch each time and must not be carried over).
if stage_name not in dvc_stages and not stage.get(
"desc", ""
).startswith("Automatically generated"):
dvc_stages[stage_name] = stage
dvc_yaml["stages"] = dvc_stages
dvc_yaml_fpath = os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml"
_dump_yaml_if_changed(dvc_yaml, dvc_yaml_fpath, verbose=verbose)
# DVC errors out (FileIsGitIgnored) if dvc.lock is Git-ignored, so make
# sure it isn't---this keeps both status and pipeline runs working
calkit.dvc.ensure_dvc_lock_not_ignored(wdir)
return dvc_stages
def get_concurrent_scheduler_stages(ck_info: dict) -> list[str]:
"""Return iterated scheduler stages eligible for concurrent submission.
These are the ``iterate_over`` stages whose outer environment is a
SLURM/PBS env. Their iterations are independent jobs, so Calkit submits
them all at once and lets the cluster's own scheduler queue them; the
scheduler, not Calkit, decides how many run simultaneously. Non-iterated
stages keep DVC's normal serial repro.
"""
from calkit.environments import COMPOSITE_ENV_SEP
environments = ck_info.get("environments", {})
stages = ck_info.get("pipeline", {}).get("stages", {})
result: list[str] = []
for name, stage in stages.items():
if not isinstance(stage, dict) or not stage.get("iterate_over"):
continue
env_name = stage.get("environment", "")
outer = env_name.split(COMPOSITE_ENV_SEP)[0]
env = environments.get(outer, {})
if env.get("kind") not in ("slurm", "pbs"):
continue
result.append(name)
return result
def get_matrix_item_targets(
stage_name: str, wdir: str | None = None
) -> tuple[list[str], list[str]]:
"""Resolve a matrix stage into its item targets and upstream targets.
Uses the compiled DVC graph so DVC's own matrix naming and dependency
resolution are the source of truth. Returns ``(item_targets,
upstream_targets)`` where ``item_targets`` address each iteration
(e.g. ``stage@a``) and ``upstream_targets`` are every stage the items
depend on transitively, excluding the items themselves. In DVC's graph an
edge points from a stage to the stage it depends on, so a node's
dependencies are its networkx descendants.
"""
import networkx as nx
import calkit.dvc
repo = calkit.dvc.get_dvc_repo(wdir)
prefix = stage_name + "@"
items = [
s
for s in repo.index.stages
if (getattr(s, "name", None) or "").startswith(prefix)
]
item_set = set(items)
upstreams: set = set()
for item in items:
upstreams |= nx.descendants(repo.index.graph, item)
upstreams -= item_set
# Sort for a deterministic order (upstreams come from a set).
return (
sorted(s.addressing for s in items),
sorted(s.addressing for s in upstreams),
)
def reproduce_targets_concurrently(
targets: list[str],
max_workers: int,
extra_args: list[str] | None = None,
wdir: str | None = None,
run_one: Callable[[str], int] | None = None,
) -> dict[str, int]:
"""Reproduce DVC targets concurrently, each in its own ``dvc repro``.
Each target runs as ``dvc repro --single-item <target>`` in a separate
process, so DVC's path-level rwlock keeps disjoint matrix items safe while
they run in parallel. ``--single-item`` is required: without it two items
sharing a stale upstream would both try to build it and one would fail
with an rwlock "busy" error, so upstreams must already be built. Returns
``{target: returncode}``. ``run_one`` is injectable for testing.
"""
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor
args = extra_args or []
# Default runner shells out to an isolated `dvc repro --single-item`. Go
# through `calkit dvc` rather than `dvc` directly so Calkit's DVC patches
# (e.g. the ck:// remote scheme) are registered in the subprocess.
# `--wait-for-lock` is a top-level DVC flag (must precede `repro`): each
# subprocess briefly acquires the repo-level lock for its status check and
# dvc.lock update, and DVC's 3s default timeout is too short when many
# iterations race to acquire it at once.
def _default_run_one(target: str) -> int:
cmd = [
sys.executable,
"-m",
"calkit",
"dvc",
"--wait-for-lock",
"repro",
"--single-item",
]
cmd += args + [target]
return subprocess.run(cmd, cwd=wdir).returncode
runner = run_one if run_one is not None else _default_run_one
results: dict[str, int] = {}
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_target = {
executor.submit(runner, target): target for target in targets
}
for future, target in future_to_target.items():
results[target] = future.result()
return results
def translate_run_targets(
targets: list[str],
ck_info: dict | None = None,
) -> tuple[list[str], list[tuple[str, str | None]]]:
"""Translate calkit-style run targets to DVC-format targets.
Supported shorthand forms:
- ``subproject`` → ``subproject/dvc.yaml`` (inline) or
the wrapper stage name (isolated, parent wrapper stage)
- ``subproject:stage`` → ``subproject/dvc.yaml:stage`` (inline) or
``(sp_path, stage)`` in the second return value (isolated; must be run
directly inside the subproject directory)
Any target that does not match a known subproject is passed through
unchanged into the first return value.
Returns
-------
tuple[list[str], list[tuple[str, str | None]]]
``(parent_dvc_targets, isolated_sp_targets)`` where
``isolated_sp_targets`` is a list of ``(sp_path, stage_or_None)``
pairs that must be reproduced by chdiring into the subproject.
"""
import calkit.markdown
if ck_info is None:
ck_info = calkit.load_calkit_info()
subprojects = ck_info.get("subprojects", [])
sp_map: dict[str, dict] = {}
for sp_cfg in subprojects:
if not isinstance(sp_cfg, dict) or not sp_cfg.get("path"):
continue
sp = Path(sp_cfg["path"]).as_posix()
sp_map[sp] = sp_cfg
sp_map[Path(sp).name] = sp_cfg
# A markdown stage stands in for the stages its blocks declare, so a
# target naming it has to become the stages DVC actually knows about.
# Done before subproject handling because a Markdown path contains no
# ':' and so can't be a subproject shorthand.
md_stages = calkit.markdown.get_markdown_stages(ck_info)
if md_stages:
expanded: list[str] = []
for target in targets:
md_path = md_stages.get(target)
if md_path is None:
expanded.append(target)
continue
expanded += calkit.markdown.get_stage_names(md_path, target)
targets = expanded
parent_targets: list[str] = []
isolated_sp_targets: list[tuple[str, str | None]] = []
for target in targets:
if ":" in target:
sp_part, stage_part = target.split(":", 1)
else:
sp_part = target
stage_part = None
if sp_part not in sp_map:
parent_targets.append(target)
continue
sp_cfg = sp_map[sp_part]
sp = Path(sp_cfg["path"]).as_posix()
sp_is_isolated = os.path.isdir(os.path.join(sp, ".dvc"))
if sp_is_isolated:
if stage_part:
isolated_sp_targets.append((sp, stage_part))
else:
parent_targets.append(Path(sp).name)
else:
if stage_part:
parent_targets.append(f"{sp}/dvc.yaml:{stage_part}")
else:
parent_targets.append(f"{sp}/dvc.yaml")
return parent_targets, isolated_sp_targets
"""Pipeline-related functionality."""
import itertools
import os
import re
import warnings
from collections.abc import Callable
from pathlib import Path
from typing import Any
import typer
from pydantic import BaseModel, Field, computed_field, field_validator
import calkit
from calkit.models.iteration import expand_project_parameters
from calkit.models.pipeline import (
InputsFromStageOutputs,
LatexStage,
PathOutput,
Pipeline,
)
def _coerce_bool(value: object) -> bool:
"""Interpret a raw YAML value the same way the Stage model would.
``ck_info`` holds unvalidated YAML, so a quoted ``"false"`` (e.g.
``frozen: "false"`` or ``always_run: "false"``) is a truthy string here;
defer to Pydantic's bool parsing so status filtering matches the compiled
pipeline's behavior.
"""
from pydantic import TypeAdapter, ValidationError
try:
return TypeAdapter(bool).validate_python(value)
except ValidationError:
return bool(value)
def frozen_stage_base_names(
ck_info: dict | None = None, wdir: str | None = None
) -> set[str]:
"""Return the base names of root pipeline stages marked ``frozen``.
Shared by the various status paths so frozen stages are consistently
hidden. Iterated stages use a ``name@param`` key in DVC status, so callers
should compare against the part before ``@``.
"""
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
names: set[str] = set()
stages = ck_info.get("pipeline", {}).get("stages", {})
if isinstance(stages, dict):
for name, cfg in stages.items():
if isinstance(cfg, dict) and _coerce_bool(
cfg.get("frozen", False)
):
names.add(name)
return names
class PipelineStatus(BaseModel):
"""Current status of the project pipeline."""
has_pipeline: bool
environment_checks: dict[str, dict] = Field(default_factory=dict)
cleaned_notebooks: list[str] = Field(default_factory=list)
stale_stages: dict[str, "StaleStage"] = Field(default_factory=dict)
errors: list[str] = Field(default_factory=list)
# Git-ignored files inside directory inputs of stages that are not
# currently stale on that input: {stage_name: {input_path: [file, ...]}}.
# DVC hashes them along with the rest of the directory, but Git doesn't
# carry them to other machines, so the stage will look stale elsewhere,
# e.g., in CI. Inputs currently reported as modified are annotated on the
# StaleStage instead, so the two never overlap.
ignored_files_in_inputs: dict[str, dict[str, list[str]]] = Field(
default_factory=dict
)
def _coerce_stale_stages(cls, value):
if not isinstance(value, dict):
return {}
coerced = {}
for stage_name, stage_value in value.items():
if isinstance(stage_value, StaleStage):
coerced[stage_name] = stage_value
else:
coerced[stage_name] = StaleStage.from_status_data(stage_value)
return coerced
def stale_stage_names(self) -> list[str]:
# Pure always-run stages aren't truly stale; surface them via
# always_run_stage_names instead.
return [
name
for name, stage in self.stale_stages.items()
Xet Storage Details
- Size:
- 97.7 kB
- Xet hash:
- b8d20f7877db691a2a6ebabacdc5d523777b09ffbae13fb4345b395e102e5544
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.