Buckets:

download
raw
97.7 kB
"""Pipeline-related functionality.""" import itertools import os import re import warnings from collections.abc import Callable from pathlib import Path from typing import Any import typer from pydantic import BaseModel, Field, computed_field, field_validator import calkit from calkit.models.iteration import expand_project_parameters from calkit.models.pipeline import ( InputsFromStageOutputs, LatexStage, PathOutput, Pipeline, ) def _coerce_bool(value: object) -> bool: """Interpret a raw YAML value the same way the Stage model would. ``ck_info`` holds unvalidated YAML, so a quoted ``"false"`` (e.g. ``frozen: "false"`` or ``always_run: "false"``) is a truthy string here; defer to Pydantic's bool parsing so status filtering matches the compiled pipeline's behavior. """ from pydantic import TypeAdapter, ValidationError try: return TypeAdapter(bool).validate_python(value) except ValidationError: return bool(value) def frozen_stage_base_names( ck_info: dict | None = None, wdir: str | None = None ) -> set[str]: """Return the base names of root pipeline stages marked ``frozen``. Shared by the various status paths so frozen stages are consistently hidden. Iterated stages use a ``name@param`` key in DVC status, so callers should compare against the part before ``@``. """ if ck_info is None: ck_info = calkit.load_calkit_info(wdir=wdir) names: set[str] = set() stages = ck_info.get("pipeline", {}).get("stages", {}) if isinstance(stages, dict): for name, cfg in stages.items(): if isinstance(cfg, dict) and _coerce_bool( cfg.get("frozen", False) ): names.add(name) return names class PipelineStatus(BaseModel): """Current status of the project pipeline.""" has_pipeline: bool environment_checks: dict[str, dict] = Field(default_factory=dict) cleaned_notebooks: list[str] = Field(default_factory=list) stale_stages: dict[str, "StaleStage"] = Field(default_factory=dict) errors: list[str] = Field(default_factory=list) # Git-ignored files inside directory inputs of stages that are not # currently stale on that input: {stage_name: {input_path: [file, ...]}}. # DVC hashes them along with the rest of the directory, but Git doesn't # carry them to other machines, so the stage will look stale elsewhere, # e.g., in CI. Inputs currently reported as modified are annotated on the # StaleStage instead, so the two never overlap. ignored_files_in_inputs: dict[str, dict[str, list[str]]] = Field( default_factory=dict ) @field_validator("stale_stages", mode="before") @classmethod def _coerce_stale_stages(cls, value): if not isinstance(value, dict): return {} coerced = {} for stage_name, stage_value in value.items(): if isinstance(stage_value, StaleStage): coerced[stage_name] = stage_value else: coerced[stage_name] = StaleStage.from_status_data(stage_value) return coerced @computed_field @property def stale_stage_names(self) -> list[str]: # Pure always-run stages aren't truly stale; surface them via # always_run_stage_names instead. return [ name for name, stage in self.stale_stages.items() if not stage.always_run or stage.modified_command or stage.modified_inputs or stage.modified_outputs or stage.stale_outputs ] @computed_field @property def always_run_stage_names(self) -> list[str]: # Only list stages whose sole change indicator is always_run; stages # that also have real changes are reported under stale_stage_names. # Subproject stages are excluded: their always-changed status is a # delegation detail, not a user-meaningful always-run stage of the # parent project. return [ name for name, stage in self.stale_stages.items() if stage.always_run and not stage.is_subproject and not stage.modified_command and not stage.modified_inputs and not stage.modified_outputs and not stage.stale_outputs ] @computed_field @property def failed_environment_checks(self) -> list[str]: return sorted( [ env_name for env_name, result in self.environment_checks.items() if not result.get("success", False) ] ) @computed_field @property def is_stale(self) -> bool: # A stage whose only listed change is "always_run" is not actually # stale — it just always re-executes by design. Treat the pipeline # as stale only if some stage has real change indicators. for stage in self.stale_stages.values(): if not stage.always_run: return True if ( stage.modified_command or stage.modified_inputs or stage.modified_outputs or stage.stale_outputs ): return True return False class StaleStage(BaseModel): """Structured status information for a stale pipeline stage.""" raw_status: list | dict | str | None = None stale_outputs: list[str] = Field(default_factory=list) modified_inputs: list[str] = Field(default_factory=list) modified_outputs: list[str] = Field(default_factory=list) modified_command: bool = False always_run: bool = False # Git-ignored files inside modified directory inputs, keyed by the entry # in ``modified_inputs``. DVC hashes a directory as a whole, so an ignored # file can change the hash without anything showing in Git status; these # are the usual suspects when a stage looks stale for no visible reason. ignored_files_in_inputs: dict[str, list[str]] = Field(default_factory=dict) # True for stages that originate from a subproject (either an individual # "{sp}:{stage}" stage or a kept "{sp} (subproject)" wrapper). Subproject # wrapper stages are marked always-changed purely as a delegation # mechanism, so they should not be advertised as always-run stages of the # parent project. is_subproject: bool = False @staticmethod def _as_path_list(paths: object) -> list[str]: if isinstance(paths, str): return [paths] if isinstance(paths, list): return [str(path) for path in paths] return [] @classmethod def _collect_paths(cls, change_group: object) -> list[str]: paths = [] if isinstance(change_group, list): for item in change_group: paths.extend(cls._collect_paths(item)) return list(dict.fromkeys(paths)) if not isinstance(change_group, dict): return [] for key, values in change_group.items(): # DVC may encode path->change_type, e.g., {"foo.txt": "modified"} if isinstance(values, str) and values in { "modified", "new", "deleted", }: paths.append(str(key)) else: paths.extend(cls._as_path_list(values)) return list(dict.fromkeys(paths)) @classmethod def _collect_output_paths( cls, change_group: object ) -> tuple[list[str], list[str]]: all_paths = [] changed_paths = [] changed_statuses = {"modified"} known_statuses = changed_statuses | { "new", "deleted", "not in cache", "always changed", } if isinstance(change_group, list): for item in change_group: item_all, item_changed = cls._collect_output_paths(item) all_paths.extend(item_all) changed_paths.extend(item_changed) return list(dict.fromkeys(all_paths)), list( dict.fromkeys(changed_paths) ) if not isinstance(change_group, dict): return [], [] for key, values in change_group.items(): if isinstance(values, str) and values in known_statuses: path = str(key) all_paths.append(path) if values in changed_statuses: changed_paths.append(path) continue if isinstance(values, list): listed_paths = [str(path) for path in values] if key in known_statuses: all_paths.extend(listed_paths) if key in changed_statuses: changed_paths.extend(listed_paths) else: all_paths.extend(listed_paths) continue item_all, item_changed = cls._collect_output_paths(values) all_paths.extend(item_all) changed_paths.extend(item_changed) return list(dict.fromkeys(all_paths)), list( dict.fromkeys(changed_paths) ) @classmethod def from_status_data( cls, status_data: list | dict | str, configured_outputs: list[str] | None = None, path_prefix: str | None = None, is_subproject: bool = False, declared_always_run: bool = False, ) -> "StaleStage": modified_inputs = [] output_paths = [] modified_outputs = [] status_blocks = [] modified_command = False marker_always_changed = False if isinstance(status_data, dict): status_blocks = [status_data] elif isinstance(status_data, list): status_blocks = [ item for item in status_data if isinstance(item, dict) ] # DVC may return plain markers like ["changed command"] or # ["always changed"], possibly alongside other change blocks. if "changed command" in status_data: modified_command = True if "always changed" in status_data: marker_always_changed = True elif isinstance(status_data, str): modified_command = status_data == "changed command" marker_always_changed = status_data == "always changed" # DVC emits the "always changed" marker for two unrelated reasons: a # stage compiled with always_changed: true (the user set always_run), # or a stage that simply has no outputs to track. Only the former is a # genuine always-run stage. A never-run, output-less stage must be # reported as stale rather than silently hidden as always-run. # Subproject wrapper stages carry the marker purely as a delegation # mechanism, so they continue to count as always-run. always_run = marker_always_changed and ( declared_always_run or is_subproject ) for block in status_blocks: if "changed command" in block: changed_command_value = block.get("changed command") # Any explicit non-false value indicates the stage command # changed and outputs should be considered stale. modified_command = modified_command or ( changed_command_value is None or changed_command_value is True or bool(changed_command_value) ) modified_inputs.extend( cls._collect_paths( block.get("changed deps", block.get("deps", {})) ) ) out_paths, changed_out_paths = cls._collect_output_paths( block.get("changed outs", block.get("outs", {})) ) output_paths.extend(out_paths) modified_outputs.extend(changed_out_paths) modified_inputs = list(dict.fromkeys(modified_inputs)) output_paths = list(dict.fromkeys(output_paths)) modified_outputs = list(dict.fromkeys(modified_outputs)) # Prefix all DVC-reported paths with the subproject directory when # viewing status from the parent repo, so all paths are consistently # parent-relative. Use normpath to collapse any "../" segments that # arise from cross-subproject deps (e.g. sp + "../shared.txt"). if path_prefix: def _prefix(p: str) -> str: return Path( os.path.normpath(os.path.join(path_prefix, p)) ).as_posix() modified_inputs = [_prefix(p) for p in modified_inputs] output_paths = [_prefix(p) for p in output_paths] modified_outputs = [_prefix(p) for p in modified_outputs] configured_outputs = [str(path) for path in (configured_outputs or [])] stale_outputs = [] if modified_inputs or modified_command: stale_outputs.extend(configured_outputs) stale_outputs.extend( [path for path in output_paths if path not in modified_outputs] ) # Fallback: if DVC flagged the stage but we couldn't attribute the # change to specific paths, mark all configured outputs as stale. # Skip the fallback for pure "always_run" stages — they always # re-execute by design, not because outputs are out of date. if ( not stale_outputs and not modified_outputs and configured_outputs and not always_run ): stale_outputs.extend(configured_outputs) stale_outputs = list(dict.fromkeys(stale_outputs)) return cls( raw_status=status_data, stale_outputs=stale_outputs, modified_inputs=modified_inputs, modified_outputs=modified_outputs, modified_command=modified_command, always_run=always_run, is_subproject=is_subproject, ) def stages_are_similar(stage1: dict, stage2: dict) -> bool: """Check if two stage configurations are fundamentally the same. Compares stage kind and key parameters to determine if stages represent the same operation. Parameters ---------- stage1 : dict First stage configuration. stage2 : dict Second stage configuration. Returns ------- bool True if stages are similar, False otherwise. """ # Different kind means different stage if stage1.get("kind") != stage2.get("kind"): return False kind = stage1.get("kind") # For script stages, check script path and args if kind in [ "python-script", "julia-script", "matlab-script", "shell-script", ]: if stage1.get("script_path") != stage2.get("script_path"): return False if stage1.get("args", []) != stage2.get("args", []): return False # For notebook stages elif kind == "jupyter-notebook": if stage1.get("notebook_path") != stage2.get("notebook_path"): return False # For latex and quarto elif kind in ["latex", "quarto"]: if stage1.get("target_path") != stage2.get("target_path"): return False # For command stages, check the command elif kind in [ "command", "shell-command", "matlab-command", "julia-command", ]: if stage1.get("command") != stage2.get("command"): return False return True def _expand_matrix(input_dict: dict[str, list]) -> list[dict]: """Restructure a dictionary with list values into a list of dictionaries, where each dictionary represents a permutation of the input dictionary's values. """ keys = list(input_dict.keys()) values = list(input_dict.values()) # Create all combinations of values using itertools.product combinations = itertools.product(*values) # Create a list of dictionaries list_of_dicts = [] for combination in combinations: list_of_dicts.append(dict(zip(keys, combination))) # After expanding the matrix, flatten any nested dictionaries in the result # This handles cases where list-of-lists iterations produce dictionaries as # values, by concatenating parent and child keys (e.g., "parent.child") # so all permutations are represented as flat dictionaries final_list = [] for item in list_of_dicts: keys = list(item.keys()) vals = list(item.values()) vd = {} for key, val in zip(keys, vals): if isinstance(val, dict): for k, v in val.items(): vd[f"{key}.{k}"] = v else: vd[key] = val final_list.append(vd) return final_list def _paths_overlap(a: str, b: str) -> bool: """Return True if path `a` and path `b` overlap in the file tree. Two paths overlap if they are equal or one is an ancestor directory of the other, e.g. ``data/`` overlaps with ``data/file.csv``. """ a = a.rstrip("/") b = b.rstrip("/") if a == b: return True return b.startswith(a + "/") or a.startswith(b + "/") def _expand_dep_excluding_subprojects( dep: str, isolated_sp_paths: list[str], root: str | None = None, ) -> list[str]: """Expand a directory dep to avoid traversing into isolated subprojects. DVC cannot collect a directory dep that contains a nested DVC repo (i.e. an isolated subproject identified by a ``.dvc/`` directory). This function walks ``dep``, listing its contents and skipping any isolated subproject directory, so the returned paths collectively cover the same files without crossing into the subprojects. ``root`` is the directory relative to which ``dep`` and ``isolated_sp_paths`` are expressed (defaults to cwd). """ root_path = Path(root) if root else Path(".") dep_path = root_path / dep # Resolve once for reliable path comparison dep_resolved = dep_path.resolve() sp_set = {(root_path / sp).resolve() for sp in isolated_sp_paths} nested = [ sp for sp in sp_set if sp != dep_resolved and sp.is_relative_to(dep_resolved) ] if not nested or not dep_path.is_dir(): return [dep] result: list[str] = [] def _walk(dir_path: Path) -> None: try: children = sorted(dir_path.iterdir()) except PermissionError: return for child in children: child_resolved = child.resolve() if child_resolved in sp_set: # Skip the isolated subproject entirely continue if child.is_dir() and any( sp.is_relative_to(child_resolved) for sp in nested ): # This directory is an ancestor of an isolated SP; descend _walk(child) else: rel = child.relative_to(root_path) suffix = "/" if child.is_dir() else "" result.append(rel.as_posix() + suffix) _walk(dep_path) return result if result else [dep] def collapse_dvc_stages( stages: dict, cmd: str | None = None, wdir: str | None = None, desc: str | None = None, ) -> dict: """Collapse a dict of DVC stages into a single wrapper stage. Collects all deps and outs from ``stages``, expands matrix templates, and returns a complete DVC stage dict. ``cmd``, ``wdir``, and ``desc`` are optional; when provided they are included in the returned dict. Rules applied: - A dep that tree-overlaps any out is dropped (e.g. a folder dep that contains output files would cause a DVC conflict). - An out that tree-overlaps any remaining dep is also dropped (covers exact-match aliasing and folder/child relationships). - Outputs are wrapped with ``{cache: false, persist: true}`` so the parent doesn't double-cache files already managed by the subproject's DVC. """ all_outs: set[str] = set() all_deps: set[str] = set() for stage_cfg in stages.values(): matrix = stage_cfg.get("matrix") replacements = _expand_matrix(matrix) if matrix else [{}] for out in stage_cfg.get("outs", []): raw = out if isinstance(out, str) else list(out.keys())[0] for r in replacements: expanded = raw for var, val in r.items(): expanded = expanded.replace(f"${{item.{var}}}", str(val)) all_outs.add(Path(expanded).as_posix()) for dep in stage_cfg.get("deps", []): if not isinstance(dep, str): continue for r in replacements: expanded = dep for var, val in r.items(): expanded = expanded.replace(f"${{item.{var}}}", str(val)) all_deps.add(Path(expanded).as_posix()) external_deps = all_deps - all_outs outs_raw = sorted(all_outs) # Track whether any dep is dropped because it is a strict folder ancestor # of an output (not an exact match — those are already removed by the set # subtraction above). When this happens the wrapper cannot fully express # the input boundary, so we mark it always_changed. # A dep is a strict ancestor of an out when the out path starts with the # dep path followed by "/". The reverse (dep inside an output folder) is # just a normal internal path drop and does not require always_changed. has_ancestor_dep_drop = any( o.startswith(d.rstrip("/") + "/") for d in external_deps for o in outs_raw ) deps = sorted( d for d in external_deps if not any(_paths_overlap(d, o) for o in outs_raw) ) dep_set = set(deps) outs = [ {o: {"cache": False, "persist": True}} for o in outs_raw if not any(_paths_overlap(o, d) for d in dep_set) ] stage: dict = {} if cmd is not None: stage["cmd"] = cmd if wdir is not None: stage["wdir"] = wdir if has_ancestor_dep_drop: stage["always_changed"] = True stage["deps"] = deps stage["outs"] = outs if desc is not None: stage["desc"] = desc return stage def _status_target_matches( target: str, display_name: str, bare_name: str, subproject: str | None, stale_stage: StaleStage, ) -> bool: """Return True if a status ``target`` selects this stage. Because the full DVC status is now computed and filtered locally (rather than asking DVC to filter by target), this is where each ``target`` passed to :func:`get_status` is resolved against a single stage. A ``target`` may be a stage name in any of calkit's forms (``stage``, ``dvc.yaml:stage``, ``subproject:stage``, or a whole ``subproject``) or a repo path, which is matched against the stage's stale/modified inputs and outputs. Iterated/matrix stages use a ``name@param`` key in DVC status, so the base name (the part before ``@``) is matched too — targeting the base name selects every iteration, consistent with how ``calkit run`` expands matrix targets. """ base_name = bare_name.split("@")[0] # Stage-name matching is done against the raw target so it covers all of # calkit's stage-target forms as well as DVC's native ``dvc.yaml:stage`` # and ``<subproject>/dvc.yaml:stage`` forms (the latter is what # translate_run_targets emits for inline subprojects). stage_aliases = {display_name, bare_name, base_name} if subproject is not None: subproject_name = Path(subproject).name if target in {subproject, subproject_name}: return display_name.startswith( f"{subproject}:" ) or display_name == (f"{subproject} (subproject)") stage_aliases |= { f"{subproject}:{bare_name}", f"{subproject_name}:{bare_name}", f"{subproject}:{base_name}", f"{subproject_name}:{base_name}", f"{subproject}/dvc.yaml:{bare_name}", f"{subproject}/dvc.yaml:{base_name}", } else: stage_aliases |= { f"dvc.yaml:{bare_name}", f"dvc.yaml:{base_name}", } if target in stage_aliases: return True # Otherwise treat the target as a repo path and match it against the # stage's stale/modified inputs and outputs. A target is path-like if it # exists on disk or contains a separator; this keeps root-level paths # without separators (e.g. ``data`` or ``out.csv``) selectable, which a # separator-only heuristic would miss. A target that looks like a stage # selector (contains ``:``) but matched no stage above is not a path. if ":" in target and not os.path.exists(target): return False is_pathlike = os.path.exists(target) or "/" in target or "\\" in target if not is_pathlike: return False norm_target = Path(target).as_posix().rstrip("/") candidate_paths = ( stale_stage.stale_outputs + stale_stage.modified_inputs + stale_stage.modified_outputs ) return any(_paths_overlap(norm_target, p) for p in candidate_paths) def get_status( ck_info: dict | None = None, targets: list[str] | None = None, wdir: str | None = None, check_environments: bool = True, clean_notebooks: bool = True, compile_to_dvc: bool = True, force_env_check: bool = False, ) -> PipelineStatus: """Get pipeline status after optional prep checks. This can compile the Calkit pipeline to DVC, clean notebook outputs, check pipeline environments, then query DVC for out-of-date stages. """ import calkit.environments import calkit.markdown prev_cwd = os.getcwd() if wdir is not None: os.chdir(wdir) try: if ck_info is None: ck_info = calkit.load_calkit_info() has_pipeline = bool(ck_info.get("pipeline", {}).get("stages", {})) has_pipeline = has_pipeline or os.path.isfile("dvc.yaml") has_subprojects = bool(ck_info.get("subprojects")) result: dict[str, Any] = { "has_pipeline": has_pipeline or has_subprojects, "environment_checks": {}, "cleaned_notebooks": [], "stale_stages": {}, "errors": [], } if not has_pipeline and not has_subprojects: return PipelineStatus.model_validate(result) # Environment checking and status filtering both read stages and # environments straight off this dict, so markdown stages have to # become ordinary ones before either runs. try: ck_info = calkit.markdown.expand_ck_info(ck_info).ck_info except Exception as e: result["errors"].append( f"Failed to read markdown stages: {e.__class__.__name__}: {e}" ) return PipelineStatus.model_validate(result) if check_environments: try: env_checks = calkit.environments.check_all_in_pipeline( ck_info=ck_info, targets=targets, force=force_env_check, ) result["environment_checks"] = env_checks except Exception as e: result["errors"].append( "Failed to check pipeline environments: " f"{e.__class__.__name__}: {e}" ) return PipelineStatus.model_validate(result) failed_env_checks = [ env_name for env_name, info in env_checks.items() if not info.get("success", False) ] if failed_env_checks: return PipelineStatus.model_validate(result) if compile_to_dvc and ( ck_info.get("pipeline", {}).get("stages", {}) or ck_info.get("subprojects") ): try: to_dvc(ck_info=ck_info, write=True, manage_gitignore=False) except Exception as e: result["errors"].append( f"Failed to compile pipeline: {e.__class__.__name__}: {e}" ) return PipelineStatus.model_validate(result) if clean_notebooks and ck_info.get("pipeline", {}).get("stages", {}): try: cleaned = calkit.notebooks.clean_all_in_pipeline( ck_info=ck_info ) result["cleaned_notebooks"] = cleaned except Exception as e: result["errors"].append( "Failed to clean notebooks in pipeline: " f"{e.__class__.__name__}: {e}" ) return PipelineStatus.model_validate(result) from dvc.lock import LockError try: dvc_repo = calkit.dvc.get_dvc_repo() # calkit.dvc.core installs a filter on dvc.repo.status that drops # the noisy frozen-stage warning, so the call here stays quiet. # Ask DVC for the complete stale set, then filter locally. # DVC's own target filtering can miss stale propagation from # isolated subprojects, which makes targeted status disagree with # the full-project status view. raw_status = dvc_repo.status() raw_status = calkit.dvc.status_as_posix(raw_status) except LockError: # Another DVC process is holding the repo lock---most often a # long-running `dvc pull`/`push` or a concurrent `calkit run`. We # genuinely cannot compute status until it releases, so state what # is running plainly rather than surfacing an alarming error (e.g. # to the VS Code extension's status poller). The holder PID is # reliable here because we just failed to acquire the lock. We've # already chdir'd into wdir above, so read the lock from the cwd # rather than re-joining the (possibly relative) wdir. holder = calkit.dvc.get_dvc_lock_holder() if holder is not None and holder.get("cmd"): result["errors"].append( f"A DVC process is in progress (PID {holder['pid']}: " f"{holder['cmd']}). Status will be available once it " "finishes." ) else: result["errors"].append( "Another DVC process is holding the lock (a run or pull " "may be in progress). Status will be available once it " "finishes." ) return PipelineStatus.model_validate(result) except Exception as e: result["errors"].append( "Failed to get pipeline status from DVC: " f"{e.__class__.__name__}: {e}" ) return PipelineStatus.model_validate(result) # Isolated subprojects appear in the parent status as wrapper stages # named after the subproject directory (e.g., "dvc.yaml:sub1"). When # a wrapper stage is stale, replace it with the individual stale stages # from the subproject's own DVC so the user sees {sp}:stage_name detail. sp_by_stage_name: dict[str, str] = {} isolated_sp_paths: set[str] = set() for sp_cfg in ck_info.get("subprojects", []): if not isinstance(sp_cfg, dict) or not sp_cfg.get("path"): continue sp = Path(sp_cfg["path"]).as_posix() if os.path.isdir(os.path.join(sp, ".dvc")): sp_by_stage_name[Path(sp).name] = sp isolated_sp_paths.add(sp) # Root-level stage keys have the form "stage_name" (no dvc.yaml: prefix). stale_wrapper_keys = [ k for k in list(raw_status.keys()) if (k.split("dvc.yaml:", 1)[-1] if "dvc.yaml:" in k else k) in sp_by_stage_name ] for wrapper_key in stale_wrapper_keys: bare_wrapper = ( wrapper_key.split("dvc.yaml:", 1)[1] if "dvc.yaml:" in wrapper_key else wrapper_key ) sp = sp_by_stage_name[bare_wrapper] try: sp_dvc_repo = calkit.dvc.get_dvc_repo(sp) sp_raw_status = sp_dvc_repo.status() sp_raw_status = calkit.dvc.status_as_posix(sp_raw_status) except Exception: sp_raw_status = {} if sp_raw_status: # Sub-project has its own stale stages: replace the wrapper key # with individual stage keys so the user sees {sp}:stage_name. del raw_status[wrapper_key] sp_is_isolated = os.path.isdir(os.path.join(sp, ".dvc")) for k, v in sp_raw_status.items(): # For isolated subprojects the DVC repo is rooted at sp, so # keys are sp-relative and need the sp/ prefix. # For inline subprojects the DVC repo is the parent root, so # keys already contain the sp path (e.g. "sub1/dvc.yaml:…"). if sp_is_isolated: if "dvc.yaml:" in k: _, bare = k.split("dvc.yaml:", 1) raw_status[f"{sp}/dvc.yaml:{bare}"] = v else: raw_status[f"{sp}/dvc.yaml:{k}"] = v else: raw_status[k] = v # else: keep the wrapper key — sub-project is internally up-to-date # but the parent's dvc.lock needs refreshing. # DVC status keys have the form: # "dvc.yaml:stage_name" root pipeline # "sub1/dvc.yaml:stage_name" subproject pipeline # Parse each key into (display_name, bare_stage_name, subproject_path). # Using a display_name of "sub1/stage_name" for subproject stages avoids # collisions when a subproject stage shares a name with a root stage. raw_stale_stages: dict[str, tuple[str, str | None, list]] = {} for k, v in raw_status.items(): # Skip DVC-tracked files (.dvc entries) — they are not pipeline # stages. Stages marked ["always changed"] are kept so the user # sees them in the pipeline status output. if k.endswith(".dvc"): continue if "dvc.yaml:" in k: prefix, bare_name = k.split("dvc.yaml:", 1) subproject = prefix.rstrip("/") if prefix else None else: bare_name = k subproject = None if subproject: display_name = f"{subproject}:{bare_name}" elif bare_name in sp_by_stage_name: # Wrapper stage kept in place (sub-project internally up-to- # date but parent dvc.lock needs refreshing). display_name = f"{sp_by_stage_name[bare_name]} (subproject)" else: display_name = bare_name raw_stale_stages[display_name] = (bare_name, subproject, v) root_stages_config = ck_info.get("pipeline", {}).get("stages", {}) # Lazily load subproject ck_info for configured_outputs lookup sp_stages_config: dict[str, dict] = {} for subproject in ck_info.get("subprojects", []): if not isinstance(subproject, dict) or not subproject.get("path"): continue sp = Path(subproject["path"]).as_posix() try: sp_ck = calkit.load_calkit_info(wdir=sp) sp_stages_config[sp] = sp_ck.get("pipeline", {}).get( "stages", {} ) except Exception: pass # Build stage ordering from root dvc.yaml; subproject stages sort after. dvc_yaml_stages: list[str] = [] dvc_yaml_stage_defs: dict[str, dict] = {} if os.path.isfile("dvc.yaml"): try: with open("dvc.yaml") as f: dvc_yaml = calkit.ryaml.load(f) dvc_yaml_stage_defs = (dvc_yaml or {}).get("stages", {}) dvc_yaml_stages = list(dvc_yaml_stage_defs.keys()) except Exception: pass ordered_stale_stages = {} dvc_yaml_stage_order = { name: i for i, name in enumerate(dvc_yaml_stages) } def _stage_sort_key(display_name: str) -> tuple[int, int]: # Subproject stages always sort after root stages (bucket 1 vs 0). bare, sp, _ = raw_stale_stages[display_name] if sp is None: base = bare.split("@")[0] return ( 0, dvc_yaml_stage_order.get(base, len(dvc_yaml_stages)), ) return (1, 0) for display_name in sorted( raw_stale_stages.keys(), key=_stage_sort_key ): bare_name, subproject, status_data = raw_stale_stages[display_name] # Frozen stages are intentionally pinned: never report them as # stale (and they are never reproduced). Iterated stages have a # "name@param" bare name, so look up config by the base name too. base_name = bare_name.split("@")[0] if subproject is None: frozen_cfg = root_stages_config.get( bare_name, root_stages_config.get(base_name, {}) ) else: sp_cfg = sp_stages_config.get(subproject, {}) frozen_cfg = sp_cfg.get(bare_name, sp_cfg.get(base_name, {})) if isinstance(frozen_cfg, dict) and _coerce_bool( frozen_cfg.get("frozen", False) ): continue if subproject is None: stage_cfg = root_stages_config.get(bare_name, {}) else: stage_cfg = sp_stages_config.get(subproject, {}).get( bare_name, {} ) # DVC reports subproject stage paths relative to the repo root # (e.g., "sub1/out.txt"), but calkit.yaml stores them relative to # the subproject dir ("out.txt"). Prefix with the subproject path # so configured_outputs matches DVC's reported paths. raw_outputs = [ output.get("path", str(output)) if isinstance(output, dict) else str(output) for output in stage_cfg.get("outputs", []) ] if subproject: # Use posix separators so these match DVC's reported paths # (which are always forward-slash, repo-root-relative). Using # the OS-native separator here would, on Windows, produce a # backslash variant that escapes path dedup and shows up as a # duplicate stale output alongside DVC's posix path. configured_outputs = [ (Path(subproject) / p).as_posix() for p in raw_outputs ] else: configured_outputs = raw_outputs # For isolated subprojects, DVC reports paths relative to the # subproject dir; prefix them so all paths are parent-relative. # A stage is subproject-originating if it carries a subproject # path ("{sp}:{stage}") or is a kept wrapper for a subproject # ("{sp} (subproject)", whose bare name is a known subproject). is_subproject = ( subproject is not None or bare_name in sp_by_stage_name ) ordered_stale_stages[display_name] = StaleStage.from_status_data( status_data=status_data, configured_outputs=configured_outputs, path_prefix=subproject if subproject in isolated_sp_paths else None, is_subproject=is_subproject, declared_always_run=_coerce_bool(stage_cfg.get("always_run")) if isinstance(stage_cfg, dict) else False, ) if targets: ordered_stale_stages = { display_name: stale_stage for display_name, stale_stage in ordered_stale_stages.items() if any( _status_target_matches( target, display_name, raw_stale_stages[display_name][0], raw_stale_stages[display_name][1], stale_stage, ) for target in targets ) } # Git-ignored files inside directory inputs. DVC hashes a directory # input as a whole without consulting .gitignore, so a stray ignored # file (scratch data, an editor backup) changes the hash yet shows up # in neither Git status nor DVC's file-level status. The stage then # looks stale for no visible reason, and since Git doesn't carry the # file to other machines, a fresh checkout such as CI hashes the # directory differently and the stage flips between stale and # up-to-date depending on where it's run (calkit#1036). Scan the # compiled dvc.yaml deps rather than calkit.yaml inputs: stage kinds # add deps of their own (a notebook's include_paths, say) and dvc.yaml # is exactly what DVC hashes. Only directories are inspected, so file # deps like cleaned notebooks or environment lock files are never # considered. Files DVC tracks (e.g., an upstream stage's outputs) are # ignored by Git on purpose and are excluded, as is anything matched # by .dvcignore, which DVC skips when hashing. stale_by_base: dict[str, list[StaleStage]] = {} for display_name, stale_stage in ordered_stale_stages.items(): bare_name, subproject, _ = raw_stale_stages[display_name] # Only root stages are scanned; iterated stages are keyed # "name@param" in DVC status, so match on the base name if subproject is None: stale_by_base.setdefault(bare_name.split("@")[0], []).append( stale_stage ) ignored_files_in_inputs: dict[str, dict[str, list[str]]] = {} # Directories are shared between stages, so list each one only once. # The Git repo and DVC index are only resolved once a directory # actually needs listing, so a pipeline without directory inputs # never pays for them. ignored_files_by_dir: dict[str, list[str]] = {} git_repo = None git_root = "" dvc_out_paths: list[str] = [] for name, stage_def in dvc_yaml_stage_defs.items(): if not isinstance(stage_def, dict): continue for dep in stage_def.get("deps", []) or []: if not isinstance(dep, str): continue # Deps are relative to the stage's wdir, if it has one dep_path = Path( os.path.normpath( os.path.join(stage_def.get("wdir", ""), dep) ) ).as_posix() if not os.path.isdir(dep_path): continue if targets and not any( target in {name, f"dvc.yaml:{name}"} or _paths_overlap(Path(target).as_posix(), dep_path) for target in targets ): continue if dep_path not in ignored_files_by_dir: if git_repo is None: git_repo = calkit.git.get_repo() git_root = str(git_repo.working_tree_dir) dvc_out_paths = [ Path(os.path.relpath(out.fs_path)).as_posix() for out in dvc_repo.index.outs ] # Git prints paths relative to the repo root, which may be # above the project, so make them project-relative; -z # keeps unusual file names from being quoted listing = git_repo.git.ls_files( "-z", "--others", "--ignored", "--exclude-standard", "--", os.path.abspath(dep_path), ) files: list[str] = [] for ignored_path in listing.split("\0"): if not ignored_path: continue ignored_path = Path( os.path.relpath( os.path.join(git_root, ignored_path) ) ).as_posix() if any( ignored_path == out or ignored_path.startswith(out + "/") for out in dvc_out_paths ): continue if dvc_repo.dvcignore.is_ignored_file( os.path.abspath(ignored_path) ): continue files.append(ignored_path) ignored_files_by_dir[dep_path] = files files = ignored_files_by_dir[dep_path] if not files: continue # When DVC reports the directory (or, if the dep was expanded # around a subproject, a path inside it) as modified, annotate # that input on the stale stage; otherwise it goes in the # status-wide list as something that will bite elsewhere annotated = False for stale_stage in stale_by_base.get(name, []): for modified_input in stale_stage.modified_inputs: inside = [ p for p in files if p == modified_input or p.startswith(modified_input.rstrip("/") + "/") ] if inside: stale_stage.ignored_files_in_inputs[ modified_input ] = inside annotated = True if not annotated: ignored_files_in_inputs.setdefault(name, {})[dep_path] = ( files ) result["stale_stages"] = ordered_stale_stages return PipelineStatus( has_pipeline=result["has_pipeline"], environment_checks=result["environment_checks"], cleaned_notebooks=result["cleaned_notebooks"], stale_stages=result["stale_stages"], errors=result["errors"], ignored_files_in_inputs=ignored_files_in_inputs, ) finally: if wdir is not None: os.chdir(prev_cwd) def get_stage_for_output(path: str, ck_info: dict) -> str | None: """Return the name of the pipeline stage that produces ``path``, if any.""" target = Path(path).as_posix() stages = ck_info.get("pipeline", {}).get("stages", {}) if not isinstance(stages, dict): return None for name, stage in stages.items(): if not isinstance(stage, dict): continue for out in stage.get("outputs", []): out_path = out.get("path") if isinstance(out, dict) else out if out_path and Path(out_path).as_posix() == target: return name return None def get_output_storage_map( ck_info: dict | None = None, wdir: str | None = None, ) -> dict[str, str]: """Get a map of pipeline output paths to their explicitly-set storage. Only outputs with an explicitly-set ``storage`` key in ``calkit.yaml`` are included so that default-DVC outputs still go through auto-detection. Parameters ---------- ck_info : dict | None Calkit project info dict. Loaded from ``calkit.yaml`` if not provided. wdir : str | None Working directory. Defaults to the current working directory. Returns ------- dict[str, str] Mapping of posix file path to storage type, e.g. ``{"figures/plot.png": "git", "data/archive": "dvc-zip"}``. Plain string outputs (no explicit ``storage`` key) are not included. """ if ck_info is None: ck_info = calkit.load_calkit_info(wdir=wdir) pipeline = ck_info.get("pipeline", {}) if not pipeline: return {} stages = pipeline.get("stages", {}) result: dict[str, str] = {} for stage in stages.values(): if not isinstance(stage, dict): continue for out in stage.get("outputs", []): if isinstance(out, dict) and "path" in out and "storage" in out: result[Path(out["path"]).as_posix()] = out["storage"] return result def _dump_yaml_if_changed(data, fpath: str, verbose: bool = False) -> bool: """Write ``data`` to ``fpath`` as YAML only if it changes the file. ``calkit status`` recompiles the pipeline to DVC on every call, and the VS Code extension polls status continuously. Blindly rewriting an identical file causes two problems: (1) bumping ``dvc.yaml``'s mtime wakes the extension's file watcher, which triggers yet another status poll---an endless loop that keeps re-taking DVC's lock; and (2) the serializer formats slightly differently across platforms and ruamel versions, so the file churns in Git for no real change. Comparison is on the parsed structure, so trivial reformatting is not treated as a change. DVC keys stage staleness on ``dvc.lock`` (not on ``dvc.yaml``'s mtime), so skipping a no-op write is safe. Returns whether the file was written. """ if os.path.isfile(fpath): try: with open(fpath) as f: if (calkit.ryaml.load(f) or {}) == data: return False except Exception: # Unreadable or invalid existing YAML: fall through and overwrite. pass if verbose: typer.echo(f"Writing to {os.path.basename(fpath)}") with open(fpath, "w") as f: calkit.ryaml.dump(data, f) return True def _parse_latexmkrc_var(text: str, var: str) -> str | None: """Extract a literal ``$<var> = '...'`` assignment from latexmkrc text. Returns the assigned string for a simple literal assignment (single- or double-quoted), or ``None`` if absent or computed. Anything non-literal is intentionally left for the user rather than guessed at. """ match = re.search( r"^\s*\$" + re.escape(var) + r"""\s*=\s*['"]([^'"]*)['"]""", text, flags=re.MULTILINE, ) if match is None: return None return match.group(1) def _warn_on_latexmkrc_out_dir_mismatch( stage: LatexStage, wdir: str | None = None ) -> None: """Warn at compile time if latexmkrc ``$out_dir`` disagrees with the stage's ``output_dir``. The tracked PDF output path is derived from ``output_dir``, so a latexmkrc that redirects the PDF without a matching ``output_dir`` would leave the real PDF untracked. Because ``calkit latex build`` runs latexmk with ``-cd``, the latexmkrc's ``$out_dir`` is relative to the LaTeX source directory, whereas ``output_dir`` (like all Calkit path fields) is relative to the stage's working directory; we resolve the former into that same frame before comparing. We only read the file when a latexmkrc is referenced (rare), so there is no cost on the common path. """ if stage.latexmkrc_path is None: return # Stage path fields are relative to the stage's wdir, which in turn is # relative to the (sub)project being compiled. base = os.path.join(wdir or ".", stage.wdir) if stage.wdir else wdir or "." try: with open(os.path.join(base, stage.latexmkrc_path)) as f: text = f.read() except OSError: return rc_out_dir = _parse_latexmkrc_var(text, "out_dir") if rc_out_dir is None: # Absent or computed (non-literal): nothing reliable to compare. return source_dir = os.path.dirname(stage.target_path) # Resolve the source-relative $out_dir into the stage's path frame (the # same frame as output_dir). An empty or "." $out_dir means "next to the # source", which is what output_dir=None also means. Comparison uses native # os.path; the resolved value is converted to POSIX only for display. rc_resolved = os.path.normpath(os.path.join(source_dir or ".", rc_out_dir)) next_to_source = os.path.normpath(source_dir or ".") if stage.output_dir is None: declared = next_to_source else: declared = os.path.normpath(stage.output_dir) if rc_resolved == declared: return rc_posix = Path(rc_resolved).as_posix() if rc_resolved == next_to_source: fix = "remove output_dir (the PDF is written next to the source)" else: fix = f"set output_dir: {rc_posix}" warnings.warn( f"LaTeX stage '{stage.name}': latexmkrc '{stage.latexmkrc_path}' " f"sets $out_dir={rc_out_dir!r} (relative to the source, i.e. " f"{rc_posix!r} in the stage's path frame), but the stage's output_dir " f"is {stage.output_dir!r}. The tracked PDF output path is derived from " "output_dir, so these should agree or the compiled PDF may not be " f"tracked. {fix}.", stacklevel=2, ) def _ensure_calkit_gitattributes(wdir: str | None = None) -> bool: """Pin the line endings of Calkit's own generated files to LF. Several of them are committed *and* hashed by DVC as stage dependencies -- an environment's lock file most of all. Git rewrites text files to CRLF on checkout when ``core.autocrlf`` is on, which is the default for Git for Windows, and DVC hashes the working tree, so a Windows collaborator's checkout disagrees with everyone else's ``dvc.lock`` and the stage reads stale forever. Writing the file with LF is not enough: the rewrite happens on the way out of Git, after we are done. DVC recommends the same mechanism for cross-platform work, but applied to everything (``* text=auto eol=lf``, plus ``core.autocrlf false``); see its guide to running on Windows. This rule is deliberately narrower, since normalizing a project's own files is the project's call and doing it unasked would renormalize files nobody meant to touch. The two agree in direction, so a project that follows DVC's advice as well loses nothing. ``.gitattributes`` outranks ``core.autocrlf``, so this holds however the user has Git configured. Scoped to ``.calkit`` --- everything Calkit generates, and nothing a project writes for itself. Deliberately not narrower than that: the files here that are both committed and hashed are spread across ``env-locks`` and ``envs``, in ``.json``, ``.yml`` and ``.txt``, and an allowlist of those would silently stop covering the next one added. Ignored files inside ``.calkit`` pick up the rule too, which costs nothing, since Git never rewrites what it isn't tracking. Locks a project keeps outside ``.calkit`` --- ``uv.lock``, ``pixi.lock``, a ``.python-version`` beside an environment spec --- are not covered. They are written by their own tools, and normalizing a project's own files is the project's call; DVC's guide to running on Windows covers them. """ return _write_managed_gitignore_block( os.path.join(wdir, ".gitattributes") if wdir else ".gitattributes", marker="calkit generated files", lines=["/.calkit/** text eol=lf"], ) def _ensure_latex_aux_gitignore( stage: LatexStage, wdir: str | None = None ) -> bool: """Ensure a managed ``.gitignore`` block ignores LaTeX aux files. The block lives in the LaTeX source directory's ``.gitignore`` rather than inside the build/output directory, which may not exist at compile time and can be wiped by a clean build (``latexmk -C``). The globs are unanchored, so aux files are caught wherever latexmk writes them relative to the source -- next to it, or in an ``aux``/output subdirectory via ``$aux_dir``/``$out_dir``/``output_dir``. ``*.pdf`` is never ignored so the compiled output stays tracked. Returns True if ``.gitignore`` was modified. """ aux_globs = [ "*.aux", "*.bbl", "*.bcf", "*.blg", "*.fdb_latexmk", "*.fls", "*.lof", "*.lot", "*.nav", "*.out", "*.run.xml", "*.snm", "*.synctex.gz", "*.toc", "*.vrb", ] # Stage path fields are relative to the stage's wdir, which in turn is # relative to the (sub)project being compiled. base = os.path.join(wdir or ".", stage.wdir) if stage.wdir else wdir or "." source_dir = os.path.dirname(stage.target_path) gitignore_path = os.path.join(base, source_dir, ".gitignore") return _write_managed_gitignore_block( gitignore_path, marker="calkit latex aux files", lines=aux_globs ) def sync_markdown( ck_info: dict | None = None, wdir: str | None = None ) -> "calkit.markdown.MarkdownExpansion": """Extract what the project's Markdown files declare, and persist envs. Run as preprocessing, before environments are checked, because an environment declared in Markdown has to be in ``calkit.yaml`` before anything tries to create or enter it. """ import calkit.markdown if ck_info is None: ck_info = calkit.load_calkit_info(wdir=wdir) markdown = calkit.markdown.expand_ck_info(ck_info, wdir=wdir) if markdown.environments: _write_markdown_environments(markdown, wdir=wdir) return markdown def _write_markdown_environments( markdown: "calkit.markdown.MarkdownExpansion", wdir: str | None = None ) -> bool: """Persist Markdown-declared environments into ``calkit.yaml``. A stage's command runs ``calkit xenv -n <env>`` as a subprocess, which reads environments back off disk, so an environment that lives only in memory can't be entered. Writing it here keeps every existing environment code path working unchanged. The Markdown stays authoritative: these entries are rewritten from it on every compile, and each carries a ``description`` saying so. Returns True if ``calkit.yaml`` was modified. """ import calkit.markdown ck_yaml_path = os.path.join(wdir, "calkit.yaml") if wdir else "calkit.yaml" if not os.path.isfile(ck_yaml_path): return False with open(ck_yaml_path) as f: data = calkit.ryaml.load(f) or {} created_envs_key = ( "environments" not in data or data["environments"] is None ) if created_envs_key: data["environments"] = {} envs = data["environments"] changed = False for env_name, env in markdown.environments.items(): if envs.get(env_name) == env: continue envs[env_name] = env changed = True # An environment a file declared on an earlier compile and no longer # does---renamed, or removed---would otherwise stay behind for good. # The generated description is what marks an entry as one of these, # so a user's own environments are never touched. for env_name in list(envs): if env_name in markdown.environments: continue source = calkit.markdown.env_source_path(envs[env_name]) if source is not None and source in markdown.markdown_paths: del envs[env_name] changed = True if changed: # Separate a newly created 'environments' section from what comes # before it, the way the rest of the file is written if created_envs_key and hasattr( data, "yaml_set_comment_before_after_key" ): try: data.yaml_set_comment_before_after_key( "environments", before="\n" ) except Exception: pass _dump_yaml_if_changed(data, ck_yaml_path) return changed def _write_managed_gitignore_block( gitignore_path: str, marker: str, lines: list[str] ) -> bool: """Create or update a marker-delimited block in a ``.gitignore`` file. The block is replaced in place if it already exists (so it stays free of duplicate/stale entries across recompiles) and appended otherwise. Returns True if the file was modified. """ begin = f"# >>> {marker} (managed by calkit) >>>" end = f"# <<< {marker} <<<" block = "\n".join([begin] + lines + [end]) existing = "" if os.path.isfile(gitignore_path): with open(gitignore_path) as f: existing = f.read() block_re = re.compile( re.escape(begin) + r".*?" + re.escape(end), flags=re.DOTALL ) if block_re.search(existing): updated = block_re.sub(lambda _: block, existing) if updated == existing: return False with open(gitignore_path, "w") as f: f.write(updated) return True parent = os.path.dirname(gitignore_path) if parent: os.makedirs(parent, exist_ok=True) separator = ( "" if not existing else ("\n" if existing.endswith("\n") else "\n\n") ) with open(gitignore_path, "a") as f: f.write(separator + block + "\n") return True def _ref_resolver( wdir: str | None, paths: list[str] | None = None ) -> Callable[[str], str] | None: """Return something that turns a Git revision into its commit hash. Stages whose inputs are revisions rather than files put the resolved hash in their command, so DVC can see when a branch has moved. None when there's no repo to ask, in which case those stages fall back to running every time. """ try: repo = calkit.git.get_repo(wdir) except Exception: return None def resolve(ref: str) -> str: sha = calkit.git.resolve_ref(repo, ref) if sha is None: # A revision that isn't there at all compiles as written; the # command reports it properly when the stage runs return ref # The commit this revision last changed the document in describes # the same document as the tip does, and doesn't move when # something else is committed return calkit.git.last_change(repo, sha, paths or []) or sha return resolve def _stage_ref_resolver( wdir: str | None, stage ) -> Callable[[str], str] | None: """A resolver that pins revisions by what this stage reads.""" paths = [p for p in stage.dvc_deps if not p.startswith(".calkit/")] return _ref_resolver(wdir, paths) def to_dvc( ck_info: dict | None = None, wdir: str | None = None, write: bool = False, verbose: bool = False, manage_gitignore: bool = True, ) -> dict: """Compile a Calkit pipeline to a DVC pipeline. If a project has subprojects, their dvc.yaml files are compiled (and written if write=True) recursively. DVC's root repro auto-discovers all dvc.yaml files in the tree, so no synthetic parent stages are needed. Returns a dictionary of DVC stages for the `stages` key of a dvc.yaml. Returns ``{}`` when the project has no pipeline and no subprojects (e.g. a subproject that is a library with no defined pipeline stages). """ import calkit.dvc.zip import calkit.markdown from calkit.environments import get_env_input_paths, get_env_lock_fpath if ck_info is None: ck_info = calkit.load_calkit_info(wdir=wdir) if "pipeline" not in ck_info and "subprojects" not in ck_info: return {} # Replace markdown stages with the stages and environments their blocks # declare, before anything else reads this, so env locks, scheduler # options and iteration need no knowledge of Markdown. markdown = calkit.markdown.expand_ck_info(ck_info, wdir=wdir) ck_info = markdown.ck_info if write and markdown.environments: _write_markdown_environments(markdown, wdir=wdir) # Everything Markdown derives is rewritten on every compile, so it # stays out of Git: the extracted scripts, and the spec file and # virtualenv of any environment the Markdown declares. One managed # block covers them all, which avoids a git check-ignore subprocess # per path. if write and manage_gitignore and markdown.script_paths: # Specs and locks are committed---that is how a Calkit project # records its environments---and so is the tooling's own bootstrap # (renv's activate.R and .Rprofile, which renv expects in version # control). What can't be committed is the installed environment # itself: it is large and holds absolute paths from the machine # that built it. renv ignores its own library via renv/.gitignore; # a virtualenv has nothing equivalent, so name it here. def _env_dir(path: str) -> str: # An environment described by a spec file in the project # directory has no directory of its own, and its virtualenv # sits beside that file return Path(os.path.dirname(path) or ".").as_posix() env_dirs = set() for env in markdown.environments.values(): env_dirs.add(_env_dir(env["path"])) # An environment the Markdown only references can live under # .calkit/envs too---that is where xr puts one it builds from the # code's imports---and its virtualenv needs ignoring just the same. for env in (ck_info.get("environments") or {}).values(): if not isinstance(env, dict) or not env.get("path"): continue env_dir = _env_dir(env["path"]) if env_dir.startswith(".calkit/envs/"): env_dirs.add(env_dir) ignore_lines = ["/.calkit/markdown/"] + [ "/.venv/" if env_dir == "." else f"/{env_dir}/.venv/" for env_dir in sorted(env_dirs) ] _write_managed_gitignore_block( os.path.join(wdir or ".", ".gitignore"), marker="calkit markdown derived files", lines=ignore_lines, ) # Compile subproject pipelines recursively. # For isolated subprojects (those with their own .dvc/ directory), DVC # won't cross the .dvc/ boundary during --all-pipelines discovery, so we # generate a single wrapper stage per isolated subproject in the parent # dvc.yaml. The wrapper's deps/outs capture the I/O boundary so DVC can # order execution correctly across subprojects. # For inline subprojects (no .dvc/ dir), DVC discovers them automatically # via --all-pipelines so no wrapper stage is needed. wrapper_stages: dict[str, dict] = {} isolated_sp_paths: list[str] = [] for subproject in ck_info.get("subprojects", []): if not isinstance(subproject, dict) or not subproject.get("path"): raise ValueError("Subprojects must have a 'path' defined") sp = Path(subproject["path"]).as_posix() if not os.path.isdir(sp): raise NotADirectoryError(f"Subproject path '{sp}' does not exist") sp_is_isolated = os.path.isdir(os.path.join(sp, ".dvc")) # Always compile the subproject's dvc.yaml recursively. # Inline subprojects need their dvc.yaml written so DVC discovers them # via --all-pipelines; isolated subprojects get a wrapper stage instead. sp_dvc_stages = to_dvc( wdir=sp, write=write, verbose=verbose, manage_gitignore=manage_gitignore, ) if not sp_is_isolated: continue isolated_sp_paths.append(sp) if not sp_dvc_stages: warnings.warn( f"Subproject '{sp}' has no pipeline stages defined; " "no wrapper stage will be created.", stacklevel=2, ) continue sp_stage_name = Path(sp).name wrapper = collapse_dvc_stages( sp_dvc_stages, cmd="calkit dvc repro", wdir=sp, desc=( f"Automatically generated wrapper for subproject '{sp}'. " "Changes made here will be overwritten." ), ) # calkit.yaml is the source of truth for the subproject pipeline; # tracking it ensures the wrapper goes stale when the pipeline # definition changes. if "calkit.yaml" not in wrapper["deps"]: wrapper["deps"] = sorted(wrapper["deps"] + ["calkit.yaml"]) wrapper_stages[sp_stage_name] = wrapper if "pipeline" not in ck_info: if write and wrapper_stages: dvc_yaml_fpath = ( os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml" ) existing = {} if os.path.isfile(dvc_yaml_fpath): with open(dvc_yaml_fpath) as f: existing = calkit.ryaml.load(f) or {} existing.setdefault("stages", {}).update(wrapper_stages) _dump_yaml_if_changed(existing, dvc_yaml_fpath) return wrapper_stages # Detect stages with the old ``slurm:`` field from the raw data, since # validation renames it to ``scheduler`` and the validated stages no # longer say which spelling they were written with. raw_stages = ck_info.get("pipeline", {}).get("stages", {}) _pre_validate_slurm_stages = [ sname for sname, sdata in raw_stages.items() if "slurm" in sdata ] try: pipeline = Pipeline.model_validate(ck_info["pipeline"]) except Exception as e: raise ValueError(f"Pipeline is not defined properly: {e}") conflicts = set(wrapper_stages) & set(pipeline.stages) if conflicts: raise ValueError( f"Subproject wrapper stage name(s) conflict with parent pipeline " f"stage name(s): {sorted(conflicts)}. Rename the subproject " f"directory or the conflicting stage(s)." ) # Seed parent stages with wrapper stages so the isolated subproject # stages appear in dvc.yaml before the parent's own stages. dvc_stages: dict = dict(wrapper_stages) # Read existing dvc.yaml now so we can clean up stale .gitignore entries # when stage outputs are renamed or removed if write: dvc_yaml_path = os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml" if os.path.isfile(dvc_yaml_path): with open(dvc_yaml_path) as f: existing_dvc_yaml = calkit.ryaml.load(f) else: existing_dvc_yaml = {} if existing_dvc_yaml is None: existing_dvc_yaml = {} existing_dvc_stages = existing_dvc_yaml.get("stages", {}) else: existing_dvc_stages = {} # First, gather up any env lock paths we might need for DVC deps. # Include both inner and outer environments so SLURM/PBS env locks are # picked up when used as the outer env in a composite environment. used_envs: set[str] = set() for stage in pipeline.stages.values(): used_envs.add(stage.inner_environment) used_envs.add(stage.outer_environment) env_lock_fpaths: dict[str, list[str]] = {} environments = ck_info.get("environments", {}) for env_name, env in environments.items(): if env_name not in used_envs: continue # An env that runs setup commands can name the files they read---a # setup script it sources, most often. Those are inputs to every # stage using the env, in the same way its lock file is: editing # how a build is set up changes what the build produces. Declared # rather than parsed out of the commands, since a shell command # doesn't reliably say which of its words is a path. Only the kinds # with a ``default_setup``: a Docker env's ``inputs`` are files # copied into the image, and their checksums already ride along in # its lock file, which stages depend on. if env.get("kind") in ("system", "slurm", "pbs"): for env_input in get_env_input_paths(env, env_name): env_lock_fpaths.setdefault(env_name, []).append( Path(env_input).as_posix() ) lock_fpath = get_env_lock_fpath( env=env, env_name=env_name, as_posix=True, for_dvc=True, wdir=wdir ) if lock_fpath is None: continue # get_env_lock_fpath prefixes wdir for most env types so the returned # path is parent-relative (e.g., "sub1/.calkit/env-locks/main"). # When writing a subproject dvc.yaml, deps must be relative to that # file's directory, so strip the wdir prefix if present. if wdir: try: lock_fpath = Path(lock_fpath).relative_to(wdir).as_posix() except ValueError: pass env_lock_fpaths.setdefault(env_name, []).append(lock_fpath) # A uv environment's interpreter comes from a .python-version next # to its spec, not from the lock, which records only a # requires-python floor. Without it as a dependency, changing the # pin would leave every stage in this environment looking current. if env.get("kind") == "uv": spec_dir = os.path.dirname(env.get("path", "")) pv_path = Path( os.path.join(spec_dir, ".python-version") ).as_posix() pv_full = os.path.join(wdir, pv_path) if wdir else pv_path if os.path.isfile(pv_full): env_lock_fpaths[env_name].append(pv_path) # An environment that installs the project itself---by pointing at # the project's own spec file, or by naming the project root as a # path dependency---is a pointer to the working tree, so the lock # says nothing about the code it resolves to: the source has to be # a dependency in its own right or editing the package would leave # its stages looking current. language = calkit.markdown.env_local_source_language(env, wdir=wdir) if language is not None: for source_path in calkit.markdown.local_package_source_paths( language, wdir=wdir ): if source_path not in env_lock_fpaths[env_name]: env_lock_fpaths[env_name].append(source_path) project_params = expand_project_parameters(ck_info.get("parameters", {})) # Convert legacy sbatch stages to shell-script + scheduler and rename # old `slurm:` stage fields to `scheduler:`, updating calkit.yaml # in-place when write=True so the file stays in sync. converted_stages = pipeline.convert_sbatch_stages() slurm_field_stages = [ sname for sname in _pre_validate_slurm_stages if sname not in converted_stages ] if write and (converted_stages or slurm_field_stages): for sname in converted_stages: typer.echo( f"Converted legacy 'sbatch' stage '{sname}' to 'shell-script' " "with 'scheduler' options in calkit.yaml" ) for sname in slurm_field_stages: typer.echo( f"Renamed 'slurm' field to 'scheduler' in stage '{sname}' " "in calkit.yaml" ) ck_yaml_path = ( os.path.join(wdir, "calkit.yaml") if wdir else "calkit.yaml" ) with open(ck_yaml_path) as _f: ck_yaml_data = calkit.ryaml.load(_f) or {} for sname, sdata in converted_stages.items(): ck_yaml_data["pipeline"]["stages"][sname] = sdata for sname in slurm_field_stages: stage_data = ck_yaml_data["pipeline"]["stages"][sname] stage_data["scheduler"] = stage_data.pop("slurm") _dump_yaml_if_changed(ck_yaml_data, ck_yaml_path) # Validate and initialize scheduler options (SLURM/PBS) for stages # whose outer environment is a job scheduler. Env defaults are applied # at job-submission time, not here. pipeline.set_stage_scheduler_options(environments=environments) # Tell procedure stages where their procedure is written down, which # only calkit.yaml knows pipeline.resolve_procedure_paths(procedures=ck_info.get("procedures", {})) if write: # The resolved setup commands each stage runs, written beside the # pipeline so the compiled command can name a path instead of # carrying shell-quoted text, which no single quoting survives on # both cmd.exe and a POSIX shell. Written after the merge above, # which is what works out the list. setup_files = [ path for stage in pipeline.stages.values() if (path := stage.write_setup_file(wdir=wdir)) is not None ] # Not committed: unlike an import's lock, this is derived from # calkit.yaml and rewritten by every compile, which is to say by # every 'calkit run' and 'calkit status' -- the same reason a # cleaned notebook isn't committed either. if manage_gitignore and setup_files: _write_managed_gitignore_block( os.path.join(wdir, ".gitignore") if wdir else ".gitignore", marker="calkit stage setup", lines=["/.calkit/stage-setup/"], ) # Only when there is a lock file to protect. A project with no # environment that locks under .calkit has nothing whose hash a # line-ending rewrite could change, and shouldn't get a # .gitattributes it never needed. if manage_gitignore and any( Path(p).as_posix().startswith(".calkit/env-locks/") for paths in env_lock_fpaths.values() for p in paths ): _ensure_calkit_gitattributes(wdir=wdir) # Ensure environment lock files are set as stage inputs if necessary pipeline.ensure_env_lock_paths_are_inputs(env_lock_fpaths=env_lock_fpaths) # Now convert Calkit stages into DVC stages for stage_name, stage in pipeline.stages.items(): # If this stage is a Jupyter notebook stage, we need to update its # parameters if any reference project-level parameters if stage.kind == "jupyter-notebook": stage.update_parameters(params=project_params) dvc_stage = stage.to_dvc() # Check if this stage iterates, which means we should create a matrix # stage if stage.iterate_over is not None: # Process a list of iterations into a DVC matrix stage # Initialize a DVC matrix dvc_matrix = {} # Initialize a dict for doing string formatting on the DVC stage format_dict = {} for n, iteration in enumerate(stage.iterate_over): arg_name = iteration.arg_name exp_vals = iteration.expand_values(params=project_params) if isinstance(arg_name, list): dvc_arg_name = f"_arg{n}" for arg_name_i in arg_name: item_string = f"${{item.{dvc_arg_name}.{arg_name_i}}}" format_dict[arg_name_i] = item_string else: dvc_arg_name = arg_name format_dict[arg_name] = f"${{item.{arg_name}}}" dvc_matrix[dvc_arg_name] = exp_vals try: cmd = dvc_stage["cmd"] cmd = cmd.format(**format_dict) dvc_stage["cmd"] = cmd except Exception as e: raise ValueError( ( f"Failed to format cmd '{cmd}': " f"{e.__class__.__name__}: {e}" ) ) formatted_deps = [] formatted_outs = [] for dep in dvc_stage.get("deps", []): try: formatted_deps.append(dep.format(**format_dict)) except Exception as e: raise ValueError( ( f"Failed to format dep '{dep}' with " f"'{format_dict}': " f"{e.__class__.__name__}: {e}" ) ) for out in dvc_stage.get("outs", []): try: if isinstance(out, dict): formatted_outs.append( { str(list(out.keys())[0]).format( **format_dict ): dict(list(out.values())[0]) } ) else: formatted_outs.append(out.format(**format_dict)) except Exception as e: raise ValueError( ( f"Failed to format out '{out}' with " f"'{format_dict}': " f"{e.__class__.__name__}: {e}" ) ) dvc_stage["deps"] = formatted_deps dvc_stage["outs"] = formatted_outs dvc_stage["matrix"] = dvc_matrix # Expand any directory deps that contain isolated subprojects so DVC # doesn't try to traverse into their .dvcignore files if isolated_sp_paths: expanded_deps = [] for dep in dvc_stage.get("deps", []): expanded_deps.extend( _expand_dep_excluding_subprojects( dep, isolated_sp_paths, root=wdir ) ) dvc_stage["deps"] = expanded_deps # Add a description to the DVC stage desc = ( f"Automatically generated from the '{stage_name}' stage " "in calkit.yaml. Changes made here will be overwritten." ) dvc_stage["desc"] = desc dvc_stages[stage_name] = dvc_stage # A Calkit stage can compile into more than one DVC stage when the # work has genuinely different inputs # Per stage, since which commits count depends on what the stage # reads. Never cached across calls: a resolver is bound to one # repo, and a process can compile pipelines in several. for extra_name, extra_stage in stage.extra_dvc_stages( resolve_ref=_stage_ref_resolver(wdir, stage) ).items(): # Raised rather than worked around with a suffix: a generated # name is addressable (calkit run <name>) and is the stage's # identity in dvc.lock, so a name that shifted when the # pipeline changed would cause spurious re-runs and orphaned # lock entries. Renaming the conflicting stage is a one-word # fix; an unpredictable name isn't. if extra_name in pipeline.stages or extra_name in dvc_stages: raise ValueError( f"Stage '{stage_name}' generates a stage named " f"'{extra_name}', which already exists; rename the " "conflicting stage" ) dvc_stages[extra_name] = extra_stage # Check for any outputs that should be ignored/unignored. # Skipped when manage_gitignore=False (e.g., status checks) to avoid # spawning a git subprocess per output (~100 ms each). if write and manage_gitignore: repo = calkit.git.get_repo(wdir) # Ensure we catch any Jupyter Notebook outputs outputs = stage.outputs.copy() if stage.kind == "jupyter-notebook": outputs += stage.notebook_outputs elif stage.kind == "marimo-html-wasm": outputs += stage.app_outputs sched_log = stage.scheduler_log_output if sched_log is not None: outputs.append(sched_log) # Build the set of current DVC output paths so we can detect stale # .gitignore entries from the previous version of the stage, # including synthesized outputs like LaTeX PDFs current_out_paths = set(calkit.dvc.out_paths_from_stage(dvc_stage)) # If this stage already existed, un-ignore any outputs that have # been renamed or removed so .gitignore does not accumulate stale # entries (e.g., after a capitalization change in the path) old_stage = existing_dvc_stages.get(stage_name, {}) for old_path in calkit.dvc.out_paths_from_stage(old_stage): if old_path not in current_out_paths: calkit.git.ensure_path_is_not_ignored(repo, path=old_path) # Deal with any gitignore changes necessary for out in outputs: if isinstance(out, PathOutput) and out.storage is None: calkit.git.ensure_path_is_ignored(repo, path=out.path) elif isinstance(out, PathOutput) and out.storage == "git": calkit.git.ensure_path_is_not_ignored(repo, path=out.path) if out.path.endswith(".ipynb"): # A notebook stage's storage is declared here, in the # pipeline, so a clean filter installed in the clone # (nbstripout) must not overrule it by stripping the # outputs back out on the way into Git. That would # commit bytes that don't match what DVC hashed, with # nothing showing as modified locally. calkit.git.ensure_path_is_not_filtered( repo, path=out.path ) elif isinstance(out, PathOutput) and out.storage == "dvc-zip": calkit.git.ensure_path_is_ignored(repo, path=out.path) calkit.dvc.zip.add(out.path, is_stage_output=True) # For LaTeX stages, warn on a latexmkrc/output_dir mismatch and keep # the generated aux files out of Git via a managed .gitignore block. # Globs (not per-file git check-ignore) keep this cheap for large # pipelines. if write and isinstance(stage, LatexStage): _warn_on_latexmkrc_out_dir_mismatch(stage, wdir=wdir) if manage_gitignore: _ensure_latex_aux_gitignore(stage, wdir=wdir) # Now process any inputs from stage outputs for stage_name, stage in pipeline.stages.items(): for i in stage.inputs: if isinstance(i, InputsFromStageOutputs): dvc_outs = dvc_stages[i.from_stage_outputs]["outs"] for out in dvc_outs: if out not in dvc_stages[stage_name]["deps"]: # Handle cases where outs are from a matrix, # in which case this output could become a list of # outputs if isinstance(out, dict): out = list(out.keys())[0] if "${item." in out: extra_outs = [] dvc_matrix = dvc_stages[i.from_stage_outputs][ "matrix" ] replacements = _expand_matrix(dvc_matrix) for r in replacements: out_i = out for var_name, var_val in r.items(): out_i = out_i.replace( f"${{item.{var_name}}}", str(var_val), ) extra_outs.append(out_i) for out_i in extra_outs: if out_i not in dvc_stages[stage_name]["deps"]: dvc_stages[stage_name]["deps"].append( out_i ) else: dvc_stages[stage_name]["deps"].append(out) if write: dvc_yaml = existing_dvc_yaml existing_stages = existing_dvc_stages for stage_name, stage in existing_stages.items(): # Preserve any existing stage not already in the new compilation # and not auto-generated (auto-generated stages are re-emitted # from scratch each time and must not be carried over). if stage_name not in dvc_stages and not stage.get( "desc", "" ).startswith("Automatically generated"): dvc_stages[stage_name] = stage dvc_yaml["stages"] = dvc_stages dvc_yaml_fpath = os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml" _dump_yaml_if_changed(dvc_yaml, dvc_yaml_fpath, verbose=verbose) # DVC errors out (FileIsGitIgnored) if dvc.lock is Git-ignored, so make # sure it isn't---this keeps both status and pipeline runs working calkit.dvc.ensure_dvc_lock_not_ignored(wdir) return dvc_stages def get_concurrent_scheduler_stages(ck_info: dict) -> list[str]: """Return iterated scheduler stages eligible for concurrent submission. These are the ``iterate_over`` stages whose outer environment is a SLURM/PBS env. Their iterations are independent jobs, so Calkit submits them all at once and lets the cluster's own scheduler queue them; the scheduler, not Calkit, decides how many run simultaneously. Non-iterated stages keep DVC's normal serial repro. """ from calkit.environments import COMPOSITE_ENV_SEP environments = ck_info.get("environments", {}) stages = ck_info.get("pipeline", {}).get("stages", {}) result: list[str] = [] for name, stage in stages.items(): if not isinstance(stage, dict) or not stage.get("iterate_over"): continue env_name = stage.get("environment", "") outer = env_name.split(COMPOSITE_ENV_SEP)[0] env = environments.get(outer, {}) if env.get("kind") not in ("slurm", "pbs"): continue result.append(name) return result def get_matrix_item_targets( stage_name: str, wdir: str | None = None ) -> tuple[list[str], list[str]]: """Resolve a matrix stage into its item targets and upstream targets. Uses the compiled DVC graph so DVC's own matrix naming and dependency resolution are the source of truth. Returns ``(item_targets, upstream_targets)`` where ``item_targets`` address each iteration (e.g. ``stage@a``) and ``upstream_targets`` are every stage the items depend on transitively, excluding the items themselves. In DVC's graph an edge points from a stage to the stage it depends on, so a node's dependencies are its networkx descendants. """ import networkx as nx import calkit.dvc repo = calkit.dvc.get_dvc_repo(wdir) prefix = stage_name + "@" items = [ s for s in repo.index.stages if (getattr(s, "name", None) or "").startswith(prefix) ] item_set = set(items) upstreams: set = set() for item in items: upstreams |= nx.descendants(repo.index.graph, item) upstreams -= item_set # Sort for a deterministic order (upstreams come from a set). return ( sorted(s.addressing for s in items), sorted(s.addressing for s in upstreams), ) def reproduce_targets_concurrently( targets: list[str], max_workers: int, extra_args: list[str] | None = None, wdir: str | None = None, run_one: Callable[[str], int] | None = None, ) -> dict[str, int]: """Reproduce DVC targets concurrently, each in its own ``dvc repro``. Each target runs as ``dvc repro --single-item <target>`` in a separate process, so DVC's path-level rwlock keeps disjoint matrix items safe while they run in parallel. ``--single-item`` is required: without it two items sharing a stale upstream would both try to build it and one would fail with an rwlock "busy" error, so upstreams must already be built. Returns ``{target: returncode}``. ``run_one`` is injectable for testing. """ import subprocess import sys from concurrent.futures import ThreadPoolExecutor args = extra_args or [] # Default runner shells out to an isolated `dvc repro --single-item`. Go # through `calkit dvc` rather than `dvc` directly so Calkit's DVC patches # (e.g. the ck:// remote scheme) are registered in the subprocess. # `--wait-for-lock` is a top-level DVC flag (must precede `repro`): each # subprocess briefly acquires the repo-level lock for its status check and # dvc.lock update, and DVC's 3s default timeout is too short when many # iterations race to acquire it at once. def _default_run_one(target: str) -> int: cmd = [ sys.executable, "-m", "calkit", "dvc", "--wait-for-lock", "repro", "--single-item", ] cmd += args + [target] return subprocess.run(cmd, cwd=wdir).returncode runner = run_one if run_one is not None else _default_run_one results: dict[str, int] = {} with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_target = { executor.submit(runner, target): target for target in targets } for future, target in future_to_target.items(): results[target] = future.result() return results def translate_run_targets( targets: list[str], ck_info: dict | None = None, ) -> tuple[list[str], list[tuple[str, str | None]]]: """Translate calkit-style run targets to DVC-format targets. Supported shorthand forms: - ``subproject`` → ``subproject/dvc.yaml`` (inline) or the wrapper stage name (isolated, parent wrapper stage) - ``subproject:stage`` → ``subproject/dvc.yaml:stage`` (inline) or ``(sp_path, stage)`` in the second return value (isolated; must be run directly inside the subproject directory) Any target that does not match a known subproject is passed through unchanged into the first return value. Returns ------- tuple[list[str], list[tuple[str, str | None]]] ``(parent_dvc_targets, isolated_sp_targets)`` where ``isolated_sp_targets`` is a list of ``(sp_path, stage_or_None)`` pairs that must be reproduced by chdiring into the subproject. """ import calkit.markdown if ck_info is None: ck_info = calkit.load_calkit_info() subprojects = ck_info.get("subprojects", []) sp_map: dict[str, dict] = {} for sp_cfg in subprojects: if not isinstance(sp_cfg, dict) or not sp_cfg.get("path"): continue sp = Path(sp_cfg["path"]).as_posix() sp_map[sp] = sp_cfg sp_map[Path(sp).name] = sp_cfg # A markdown stage stands in for the stages its blocks declare, so a # target naming it has to become the stages DVC actually knows about. # Done before subproject handling because a Markdown path contains no # ':' and so can't be a subproject shorthand. md_stages = calkit.markdown.get_markdown_stages(ck_info) if md_stages: expanded: list[str] = [] for target in targets: md_path = md_stages.get(target) if md_path is None: expanded.append(target) continue expanded += calkit.markdown.get_stage_names(md_path, target) targets = expanded parent_targets: list[str] = [] isolated_sp_targets: list[tuple[str, str | None]] = [] for target in targets: if ":" in target: sp_part, stage_part = target.split(":", 1) else: sp_part = target stage_part = None if sp_part not in sp_map: parent_targets.append(target) continue sp_cfg = sp_map[sp_part] sp = Path(sp_cfg["path"]).as_posix() sp_is_isolated = os.path.isdir(os.path.join(sp, ".dvc")) if sp_is_isolated: if stage_part: isolated_sp_targets.append((sp, stage_part)) else: parent_targets.append(Path(sp).name) else: if stage_part: parent_targets.append(f"{sp}/dvc.yaml:{stage_part}") else: parent_targets.append(f"{sp}/dvc.yaml") return parent_targets, isolated_sp_targets
"""Pipeline-related functionality."""
import itertools
import os
import re
import warnings
from collections.abc import Callable
from pathlib import Path
from typing import Any
import typer
from pydantic import BaseModel, Field, computed_field, field_validator
import calkit
from calkit.models.iteration import expand_project_parameters
from calkit.models.pipeline import (
InputsFromStageOutputs,
LatexStage,
PathOutput,
Pipeline,
)
def _coerce_bool(value: object) -> bool:
"""Interpret a raw YAML value the same way the Stage model would.
``ck_info`` holds unvalidated YAML, so a quoted ``"false"`` (e.g.
``frozen: "false"`` or ``always_run: "false"``) is a truthy string here;
defer to Pydantic's bool parsing so status filtering matches the compiled
pipeline's behavior.
"""
from pydantic import TypeAdapter, ValidationError
try:
return TypeAdapter(bool).validate_python(value)
except ValidationError:
return bool(value)
def frozen_stage_base_names(
ck_info: dict | None = None, wdir: str | None = None
) -> set[str]:
"""Return the base names of root pipeline stages marked ``frozen``.
Shared by the various status paths so frozen stages are consistently
hidden. Iterated stages use a ``name@param`` key in DVC status, so callers
should compare against the part before ``@``.
"""
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
names: set[str] = set()
stages = ck_info.get("pipeline", {}).get("stages", {})
if isinstance(stages, dict):
for name, cfg in stages.items():
if isinstance(cfg, dict) and _coerce_bool(
cfg.get("frozen", False)
):
names.add(name)
return names
class PipelineStatus(BaseModel):
"""Current status of the project pipeline."""
has_pipeline: bool
environment_checks: dict[str, dict] = Field(default_factory=dict)
cleaned_notebooks: list[str] = Field(default_factory=list)
stale_stages: dict[str, "StaleStage"] = Field(default_factory=dict)
errors: list[str] = Field(default_factory=list)
# Git-ignored files inside directory inputs of stages that are not
# currently stale on that input: {stage_name: {input_path: [file, ...]}}.
# DVC hashes them along with the rest of the directory, but Git doesn't
# carry them to other machines, so the stage will look stale elsewhere,
# e.g., in CI. Inputs currently reported as modified are annotated on the
# StaleStage instead, so the two never overlap.
ignored_files_in_inputs: dict[str, dict[str, list[str]]] = Field(
default_factory=dict
)
@field_validator("stale_stages", mode="before")
@classmethod
def _coerce_stale_stages(cls, value):
if not isinstance(value, dict):
return {}
coerced = {}
for stage_name, stage_value in value.items():
if isinstance(stage_value, StaleStage):
coerced[stage_name] = stage_value
else:
coerced[stage_name] = StaleStage.from_status_data(stage_value)
return coerced
@computed_field
@property
def stale_stage_names(self) -> list[str]:
# Pure always-run stages aren't truly stale; surface them via
# always_run_stage_names instead.
return [
name
for name, stage in self.stale_stages.items()

Xet Storage Details

Size:
97.7 kB
·
Xet hash:
b8d20f7877db691a2a6ebabacdc5d523777b09ffbae13fb4345b395e102e5544

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.