Buckets:
"""Functionality related to environments."""
import functools
import glob
import hashlib
import json
import os
import platform
import re
import socket
import subprocess
import tempfile
from pathlib import Path
from typing import cast
import toml
import yaml
from pydantic import BaseModel
from sqlitedict import SqliteDict
import calkit
DOCKER_ARCHS = [
"amd64",
"arm64",
"arm-v7",
"arm-v6",
"arm-v5",
"ppc64le",
"s390x",
"386",
"riscv64",
]
# Architectures Docker environments are locked for by default, so that
# moving a project between them doesn't invalidate every stage
DEFAULT_DOCKER_LOCK_ARCHS = ["amd64", "arm64"]
DEFAULT_PYTHON_VERSION = "3.14"
CONDA_VENV_ARCHS = [
"osx-arm64",
"osx-64",
"linux-aarch64",
"linux-ppc64le",
"linux-64",
"win-64",
]
ENV_CHECK_CACHE_TTL_SECONDS = 3600
# Scheduler environment keys that govern how a job is dispatched rather than
# what it computes. They are excluded from the environment lock file, so
# changing them does not invalidate cached results: pacing submissions to be
# polite to a shared queue must not force every simulation to rerun.
SCHEDULER_DISPATCH_ONLY_KEYS = {"max_concurrent_jobs"}
# Environment kinds with nothing to build or verify. Note this matches on
# ``kind``; the special ``_system`` environment is excluded by name instead,
# since names starting with an underscore are filtered out of the pipeline's
# environment list before we get here. ``system`` is not listed: checking one
# means writing its lock file, so it has to go through ``check_environment``.
# No kind is unconditionally uncheckable. A ``system`` env on another host
# comes close, but that depends on the env's host rather than its kind, and
# ``check_environment`` handles it: with nothing locked there is nothing to
# check, and locking a machine we can't observe is an error.
KINDS_NO_CHECK: list[str] = []
# Kinds whose check must not be cached. Caching exists to skip rebuilding
# something expensive, but checking a ``system`` env *is* reading the
# machine -- there is nothing to skip. Caching it means a locked property
# can change and be missed until the cache expires, which is precisely the
# drift ``lock`` exists to catch.
KINDS_NO_CACHE = ["system"]
def cacheable(env: dict) -> bool:
"""Whether an environment's check is worth remembering."""
return env.get("kind") not in KINDS_NO_CACHE
# Maps the kebab-case properties a ``system`` environment can lock onto the
# keys ``get_system_info`` returns. Not a mechanical transformation, hence
# the explicit table: note ``Rscript_version``'s capital R.
SYSTEM_LOCK_PROPERTIES = {
"os": "os",
"os-version": "os_version",
"platform": "platform",
"machine": "machine",
"processor": "processor",
"hostname": "hostname",
"machine-id": "machine_id",
"cpu-count": "cpu_count",
"memory-gb": "memory_gb",
"python-version": "python_version",
"python-implementation": "python_implementation",
"git-version": "git_version",
"docker-version": "docker_version",
"conda-version": "conda_version",
"mamba-version": "mamba_version",
"uv-version": "uv_version",
"pixi-version": "pixi_version",
"julia-version": "julia_version",
"juliaup-version": "juliaup_version",
"rscript-version": "Rscript_version",
"brew-version": "brew_version",
}
# What each lockable property means, for the published documentation. Kept
# beside the table above so the two can be checked against each other; a
# property nobody can describe is one nobody can decide whether to lock.
SYSTEM_LOCK_PROPERTY_DESCRIPTIONS = {
"os": "Operating system name, e.g. 'Linux' or 'Darwin'.",
"os-version": "Operating system release, e.g. a kernel version.",
"platform": "Full platform string, which folds in most of the above.",
"machine": "Machine architecture, e.g. 'x86_64' or 'arm64'.",
"processor": "Processor name, where the OS reports one.",
"hostname": "The machine's name. Pins results to one specific host, "
"but only by name: renaming the machine breaks the pin, and a machine "
"elsewhere with the same name satisfies it. Prefer 'machine-id'.",
"machine-id": "A stable identifier for the machine itself, read from "
"the platform. Pins results to one specific machine, and unlike "
"'hostname' survives renaming it. Declaring a 'machine_id' on the "
"environment says where to run, not that results depend on it, so "
"lock this to also rerun stages when the machine changes.",
"cpu-count": "Number of CPUs, which can change what a run produces "
"where results depend on how work was divided.",
"memory-gb": "Total memory in GB.",
"python-version": "Version of the Python running Calkit.",
"python-implementation": "Python implementation, e.g. 'CPython'.",
"git-version": "Installed Git version.",
"docker-version": "Installed Docker version.",
"conda-version": "Installed Conda version.",
"mamba-version": "Installed Mamba version.",
"uv-version": "Installed uv version.",
"pixi-version": "Installed Pixi version.",
"julia-version": "Installed Julia version.",
"juliaup-version": "Installed Juliaup version.",
"rscript-version": "Installed Rscript version.",
"brew-version": "Installed Homebrew version.",
}
# How precisely a property is worth recording. Total memory is reported as
# a bare division of bytes by 1024**3, so a machine describes itself as
# having 15.492069244384766 GB, and a firmware or kernel update that
# reserves a little more or less moves that without changing anything a
# result could depend on. Rounding it is the difference between pinning how
# much memory the machine has and pinning what it happened to report.
# Everything else is recorded as given: an OS version or a CPU count means
# exactly what it says.
SYSTEM_LOCK_PROPERTY_PRECISION = {"memory-gb": lambda v: round(float(v), 1)}
# Properties only one platform can supply, since ``get_system_info`` collects
# package manager versions per OS. Locking one from another platform raises
# in ``get_system_lock_data`` rather than recording nothing, so this table is
# documentation (and a test hook), not a second gate.
SYSTEM_LOCK_PROPERTY_PLATFORMS = {"brew-version": "Darwin"}
def _as_posix_path(path: str) -> str:
return Path(path).as_posix()
COMPOSITE_ENV_SEP = ":"
# Kinds that say *where* a stage runs rather than what it runs in, so they
# can wrap an inner runtime env as ``<outer>:<inner>``.
VALID_OUTER_ENV_KINDS = ["slurm", "pbs", "system"]
def host_is_local(host: str | None) -> bool:
"""Whether ``host`` names the machine we're running on.
Environments that name a host (``system``, ``slurm``, ``pbs``) are
declarations of where the work belongs, not instructions to connect:
when we're already on that machine there is nothing to reach out to.
A machine reports itself as a bare name or a fully qualified one
depending on how it's configured, and projects write it either way, so
the two are matched across that difference. A domain is only dropped
from one side at a time, so two different machines that share a short
name under different domains stay distinct.
"""
if not host or host == "localhost":
return True
current_host = socket.gethostname()
current_fqdn = socket.getfqdn()
if host in (current_host, current_fqdn):
return True
if "." not in host:
# A bare env host matches this machine's short name, however this
# machine happens to report itself.
if host in (
current_host.split(".")[0],
current_fqdn.split(".")[0],
):
return True
elif "." not in current_fqdn:
# A qualified env host can still name a machine that only knows its
# own short name; there is no domain here to contradict it.
if host.split(".")[0] in (current_host, current_fqdn):
return True
# Names are not the only way to write a machine down. A host given as
# an IP address never matches a hostname, and a machine reached at one
# address may call itself something else entirely -- so ask whether any
# address this host resolves to is one of ours.
return _resolves_to_this_machine(host)
@functools.lru_cache(maxsize=128)
def _host_addresses(host: str) -> tuple[tuple[int, str], ...]:
"""The addresses ``host`` resolves to, as (family, address) pairs."""
try:
infos = socket.getaddrinfo(host, None, type=socket.SOCK_DGRAM)
except (socket.gaierror, UnicodeError, OSError):
return ()
return tuple(
{(family, sockaddr[0]) for family, _, _, _, sockaddr in infos}
)
def _address_is_local(family: int, address: str) -> bool:
"""Whether an address belongs to an interface on this machine.
Binding is the question itself rather than a proxy for it: an address
can only be bound where it is actually configured, which is exactly
what "this is my address" means. Reading interfaces directly would need
a dependency and would still have to answer the same question.
"""
try:
with socket.socket(family, socket.SOCK_DGRAM) as sock:
sock.bind((address, 0))
return True
except OSError:
return False
def _resolves_to_this_machine(host: str) -> bool:
if any(
_address_is_local(family, address)
for family, address in _host_addresses(host)
):
return True
# A bare name can be one only mDNS knows, which is not in the resolver's
# search list the way a DNS domain is -- so it resolves under '.local'
# and nowhere else. This is the name macOS shows the user as theirs (in
# Sharing, and from 'scutil --get LocalHostName'), so it is the name
# they are most likely to write down, and it would otherwise be the one
# name for this machine that fails to match it.
if "." in host:
return False
return any(
_address_is_local(family, address)
for family, address in _host_addresses(host + ".local")
)
def env_is_local(env: dict) -> bool:
"""Whether an environment's machine is the one we're running on.
A declared ``machine_id`` is the answer when there is one: it is a
stronger claim than a name, so a host that resolves here while the ID
says otherwise is a machine that was rebuilt or a name that now points
somewhere else -- both cases where running here would be wrong.
That only holds while we can tell which machine this is. Where no ID
can be read, a declared one can never match, and taking that as "not
this machine" would send a user off to SSH into the box they are
sitting at; the name is what's left to go on, so it decides.
"""
declared = env.get("machine_id")
if declared:
declared = os.path.expandvars(declared)
if declared:
current = calkit.get_machine_id()
if current is not None:
return calkit.machine_ids_match(declared, current)
return host_is_local(os.path.expandvars(env.get("host") or ""))
def get_julia_packages_dir() -> str:
"""Return the Julia packages directory for the current environment."""
depot_env = os.getenv("JULIA_DEPOT_PATH", "")
first_depot = depot_env.split(os.pathsep)[0].strip() if depot_env else ""
if not first_depot:
first_depot = os.path.join("~", ".julia")
return os.path.join(os.path.expanduser(first_depot), "packages")
def _calc_dir_sig_shallow(path: str, max_depth: int = 1) -> str:
"""Calculate a lightweight signature by scanning only a few levels.
This avoids deep recursive walks of large directories while still
detecting practical state changes like added/removed packages or
artifacts.
"""
if not os.path.isdir(path):
return ""
try:
root_stat = os.stat(path)
latest_mtime = root_stat.st_mtime_ns
except OSError:
return ""
entry_count = 0
total_size = 0
stack: list[tuple[str, int]] = [(path, 0)]
while stack:
current, depth = stack.pop()
try:
with os.scandir(current) as it:
for entry in it:
try:
st = entry.stat(follow_symlinks=False)
except OSError:
continue
entry_count += 1
total_size += st.st_size
if st.st_mtime_ns > latest_mtime:
latest_mtime = st.st_mtime_ns
if depth < max_depth and entry.is_dir(
follow_symlinks=False
):
stack.append((entry.path, depth + 1))
except OSError:
continue
return f"{entry_count}-{total_size}-{latest_mtime}"
def calc_julia_depot_sig() -> str | None:
"""Calculate a cheap machine-state signature for Julia depot changes."""
packages_dir = get_julia_packages_dir()
depot_root = os.path.dirname(packages_dir)
packages_sig = _calc_dir_sig_shallow(packages_dir, max_depth=1)
artifacts_sig = _calc_dir_sig_shallow(
os.path.join(depot_root, "artifacts"), max_depth=1
)
registries_sig = _calc_dir_sig_shallow(
os.path.join(depot_root, "registries"), max_depth=1
)
if not any([packages_sig, artifacts_sig, registries_sig]):
return None
return "|".join([packages_sig, artifacts_sig, registries_sig])
def language_from_env(env: dict) -> str | None:
kind = env.get("kind")
if kind == "julia":
return "julia"
if kind == "renv":
return "r"
if kind == "matlab":
return "matlab"
if kind in ["conda", "pixi", "uv", "uv-venv", "venv"]:
return "python"
if kind == "docker" and "texlive" in env.get("image", "").lower():
return "latex"
return None
def _get_julia_version() -> str:
"""Detect the active Julia version.
Returns
-------
str
Julia version string (e.g., "1.10.1"). Defaults to "1.10" if
detection fails.
"""
try:
result = subprocess.run(
[calkit.julia.get_julia_exe(), "--version"],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0:
# Parse output like "julia version 1.10.1"
output = result.stdout.strip()
# Extract version number
parts = output.split()
for part in parts:
# Check if this part looks like a version
if part and part[0].isdigit():
# Return major.minor version
version_parts = part.split(".")
if len(version_parts) >= 2:
return f"{version_parts[0]}.{version_parts[1]}"
return "1.10"
except (subprocess.TimeoutExpired, FileNotFoundError, Exception):
# If Julia is not available or detection fails, default to 1.10
return "1.10"
def get_env_lock_dir(wdir: str | None = None) -> str:
env_lock_dir = os.path.join(".calkit", "env-locks")
if wdir is not None:
env_lock_dir = os.path.join(wdir, env_lock_dir)
return env_lock_dir
def _conda_venv_platform() -> str:
sys = platform.system().lower()
mach = platform.machine().lower()
if sys == "darwin":
return "osx-arm64" if mach in ("arm64", "aarch64") else "osx-64"
if sys == "linux":
if mach in ("arm64", "aarch64"):
return "linux-aarch64"
elif mach in ("ppc64le",):
return "linux-ppc64le"
else:
return "linux-64"
if sys == "windows":
return "win-64"
# Fallback for unusual platforms
return f"{sys}-{mach}"
def get_docker_arch() -> str:
"""Get Docker platform string (arch part only)."""
mach = platform.machine().lower()
# Map common platform.machine() outputs to Docker arch names
if mach in ("x86_64", "amd64"):
return "amd64"
elif mach in ("aarch64", "arm64"):
return "arm64"
elif mach in ("armv7l", "armv7"):
return "arm-v7"
elif mach in ("armv6l", "armv6"):
return "arm-v6"
elif mach == "ppc64le":
return "ppc64le"
elif mach == "s390x":
return "s390x"
elif mach in ("i386", "i686"):
return "386"
elif mach == "riscv64":
return "riscv64"
# Fallback
return mach
def get_all_docker_lock_fpaths(
env_name: str,
wdir: str | None = None,
as_posix: bool = True,
) -> list[str]:
"""Return docker environment lock file paths for every supported
architecture.
This intentionally excludes legacy (pre-arch) lock file locations;
legacy handling is performed separately.
"""
env_lock_dir = get_env_lock_dir(wdir=wdir)
docker_dir = os.path.join(env_lock_dir, env_name)
fpaths = [
os.path.join(docker_dir, arch + ".json") for arch in DOCKER_ARCHS
]
if as_posix:
fpaths = [Path(p).as_posix() for p in fpaths]
return fpaths
def get_all_conda_lock_fpaths(
env_name: str,
wdir: str | None = None,
as_posix: bool = True,
) -> list[str]:
"""Return conda environment lock file paths for every supported
architecture.
"""
env_lock_dir = get_env_lock_dir(wdir=wdir)
env_lock_dir = os.path.join(env_lock_dir, env_name)
fpaths = [
os.path.join(env_lock_dir, arch + ".yml") for arch in CONDA_VENV_ARCHS
]
if as_posix:
fpaths = [Path(p).as_posix() for p in fpaths]
return fpaths
def get_all_venv_lock_fpaths(
env_name: str,
wdir: str | None = None,
as_posix: bool = True,
) -> list[str]:
"""Return venv environment lock file paths for every supported
architecture.
"""
env_lock_dir = get_env_lock_dir(wdir=wdir)
venv_dir = os.path.join(env_lock_dir, env_name)
fpaths = [
os.path.join(venv_dir, arch + ".txt") for arch in CONDA_VENV_ARCHS
]
if as_posix:
fpaths = [Path(p).as_posix() for p in fpaths]
return fpaths
def _get_julia_manifest_fpath(
env_dir: str, julia_version: str | None, wdir: str | None = None
) -> str:
"""Return the Manifest path for a Julia env, preferring versioned names.
Julia 1.9+ writes ``Manifest-vMAJOR.MINOR.toml`` (e.g.
``Manifest-v1.11.toml``) alongside the default ``Manifest.toml``.
We prefer the versioned file when it exists.
"""
base_dir = env_dir or "."
full_dir = os.path.join(wdir, base_dir) if wdir else base_dir
if julia_version:
parts = julia_version.split(".")
if len(parts) >= 2:
major_minor = f"{parts[0]}.{parts[1]}"
versioned_name = f"Manifest-v{major_minor}.toml"
if os.path.isfile(os.path.join(full_dir, versioned_name)):
return os.path.join(base_dir, versioned_name)
return os.path.join(base_dir, "Manifest.toml")
# The shell a system environment's setup commands run in. Setup is the only
# thing that uses it, and bash is the default because ``source`` is a
# bashism and sourcing a site setup script is the usual reason to have any.
SYSTEM_ENV_DEFAULT_SHELL = "bash"
def system_env_locks_anything(env: dict) -> bool:
"""Return whether a ``system`` environment has anything to lock.
The machine properties under ``lock``, and a ``shell`` that isn't the
default. Not ``default_setup``: the pipeline compiler merges that into
the stage's command, so DVC already reruns the stage when it changes
and a copy here would only add a second reason. The shell isn't in the
command, so it stays.
``shell`` is compared by value rather than presence: writing
``shell: bash`` says exactly what leaving it out says, so it must not
be what decides whether stages gain a dependency.
"""
return bool(
env.get("lock")
or env.get("shell", SYSTEM_ENV_DEFAULT_SHELL)
!= SYSTEM_ENV_DEFAULT_SHELL
)
def get_env_lock_fpath(
env: dict,
env_name: str,
wdir: str | None = None,
as_posix: bool = True,
legacy: bool = False,
for_dvc: bool = False,
) -> str | None:
"""Create the environment lock file path.
If `for_dvc` is True, return the directory containing the lock file
instead of the lock file itself for Docker, venv, and conda environments,
which store a separate lock file for each platform/architecture.
"""
env_lock_dir = get_env_lock_dir(wdir=wdir)
env_kind = env.get("kind")
lock_fpath = os.path.join(env_lock_dir, env_name)
if env_kind == "docker":
if legacy:
lock_fpath += ".json"
else:
lock_fpath = os.path.join(
env_lock_dir, env_name, get_docker_arch() + ".json"
)
if for_dvc:
lock_fpath = os.path.dirname(lock_fpath)
elif env_kind == "uv":
env_dir = os.path.dirname(env.get("path", ""))
if env_dir:
lock_fpath = os.path.join(env_dir, "uv.lock")
else:
lock_fpath = "uv.lock"
elif env_kind == "pixi":
env_dir = os.path.dirname(env.get("path") or "")
if env_dir:
lock_fpath = os.path.join(env_dir, "pixi.lock")
else:
lock_fpath = "pixi.lock"
elif env_kind in ["venv", "uv-venv"]:
if legacy:
lock_fpath += ".txt"
else:
lock_fpath = os.path.join(
env_lock_dir,
env_name,
_conda_venv_platform() + ".txt",
)
if for_dvc:
lock_fpath = os.path.dirname(lock_fpath)
elif env_kind == "conda":
if legacy:
lock_fpath += ".yml"
else:
lock_fpath = os.path.join(
env_lock_dir,
env_name,
_conda_venv_platform() + ".yml",
)
if for_dvc:
lock_fpath = os.path.dirname(lock_fpath)
elif env_kind == "matlab":
lock_fpath += ".json"
elif env_kind == "julia":
env_path = env.get("path")
if env_path is None:
raise ValueError(
"Julia environments require a path pointing to Project.toml"
)
env_fname = os.path.basename(env_path)
if not env_fname == "Project.toml":
raise ValueError(
"Julia environments require a path pointing to Project.toml"
)
env_dir = os.path.dirname(env_path)
lock_fpath = _get_julia_manifest_fpath(
env_dir, env.get("julia"), wdir=wdir
)
elif env_kind == "renv":
env_path = env.get("path")
if env_path is None:
raise ValueError(
"renv environments require a path pointing to DESCRIPTION"
)
env_fname = os.path.basename(env_path)
if not env_fname == "DESCRIPTION":
raise ValueError(
"renv environments require a path pointing to DESCRIPTION"
)
# Replace DESCRIPTION with renv.lock
env_dir = os.path.dirname(env_path)
lock_fpath = os.path.join(env_dir, "renv.lock")
elif env_kind == "nix":
env_path = env.get("path")
if env_path is None:
raise ValueError(
"Nix environments require a path pointing to flake.nix"
)
env_fname = os.path.basename(env_path)
if env_fname != "flake.nix":
raise ValueError(
"Nix environments require a path pointing to flake.nix"
)
# flake.lock is generated by ``nix flake lock`` next to flake.nix.
env_dir = os.path.dirname(env_path)
lock_fpath = os.path.join(env_dir, "flake.lock")
elif env_kind == "system":
# A system env's lock file records the machine properties it
# declared it depends on, plus a ``shell`` that isn't the default.
# Not ``default_setup``: that is compiled into each stage's
# command. With neither there is nothing to depend on, so no lock
# file and no DVC dep.
if not system_env_locks_anything(env):
return None
# Written by ``write_system_env_lock`` during environment checks, and
# referenced as a DVC dep by stage compilation. Note this is the file
# itself even when ``for_dvc``: there's exactly one of them, so
# there's no reason to make a stage depend on the whole directory.
lock_fpath = os.path.join(env_lock_dir, env_name, "info.json")
elif env_kind in ("slurm", "pbs"):
# Job-scheduler envs have no external dependency manifest, so the
# "lock" is just a JSON dump of the env config. The file is
# written by ``write_scheduler_env_lock`` during environment
# checks (e.g., ``calkit check env``) and stage compilation
# references it as a DVC dep so changes invalidate cached runs.
lock_fpath = os.path.join(env_lock_dir, env_name, "info.json")
if for_dvc:
lock_fpath = os.path.dirname(lock_fpath)
else:
return
if as_posix:
lock_fpath = Path(lock_fpath).as_posix()
return lock_fpath
def write_scheduler_env_lock(
env_name: str,
env: dict,
wdir: str | None = None,
) -> str | None:
"""Write a JSON lock file for a SLURM or PBS environment.
The lock file simply contains a deterministic JSON dump of the env
config so DVC can use it as a stage dependency: when the env's
``default_options``, ``default_setup``, ``host``, etc. change, the
lock file changes and any stage that depends on it is invalidated.
Keys in ``SCHEDULER_DISPATCH_ONLY_KEYS`` are left out, since they change
only when a job is submitted, not what it computes.
Parameters
----------
env_name : str
Environment name as it appears in ``calkit.yaml``.
env : dict
Environment configuration dict.
wdir : str | None
Working directory; defaults to the current process cwd.
Returns
-------
str | None
The lock file path, already prefixed with ``wdir`` if provided, or
``None`` if the env kind has no scheduler lock file.
"""
if env.get("kind") not in ("slurm", "pbs"):
return None
# Already prefixed with wdir, so it must not be joined with it again
lock_fpath = get_env_lock_fpath(
env=env, env_name=env_name, wdir=wdir, as_posix=True
)
if lock_fpath is None:
return None
parent = os.path.dirname(lock_fpath)
if parent:
os.makedirs(parent, exist_ok=True)
lock_data = {
k: v for k, v in env.items() if k not in SCHEDULER_DISPATCH_ONLY_KEYS
}
# Record when the scheduler is mocked so switching between a mocked run
# (executed locally) and a real scheduler run changes the lock file and
# invalidates the cached result
from calkit.cli.scheduler import _mock_enabled
if _mock_enabled():
lock_data["mocked"] = True
content = json.dumps(lock_data, indent=2, sort_keys=True) + "\n"
if os.path.isfile(lock_fpath):
with open(lock_fpath, "r") as f:
existing = f.read()
if existing == content:
return lock_fpath
# newline="\n" so the file is byte-identical on every platform.
# Unlike a system env's lock, this one is a dump of the env config
# rather than of the machine, so its content is the same
# everywhere -- and it is written wherever the check runs, not
# where the scheduler lives, so text mode would let a Windows
# collaborator flip it to CRLF and back for everyone else.
with open(lock_fpath, "w", newline="\n") as f:
f.write(content)
return lock_fpath
def merge_setup_commands(
env_setup: list[str] | None,
stage_setup: list[str] | None,
mode: str = "replace",
) -> list[str]:
"""Combine an environment's default setup commands with a stage's.
``replace`` (the default) uses the environment's only when the stage
names none of its own; ``merge`` runs the environment's first, then the
stage's; ``ignore`` never runs the environment's.
The one implementation of the rule. A system env's stages have this
resolved when the pipeline is compiled, so the merged list ends up in
the stage's command; a scheduler env's is resolved by ``calkit
scheduler batch`` when the job script is written, which is the last
moment before that kind of stage runs.
"""
stage_cmds = [c for c in (stage_setup or []) if c.strip()]
env_cmds = [c for c in (env_setup or []) if c.strip()]
if mode == "merge":
return env_cmds + stage_cmds
if mode == "replace" and not stage_cmds:
return env_cmds
return stage_cmds
# Warned once per process, not once per reader: compiling a pipeline reads
# an environment's inputs several times -- the DVC dep list, the
# environment check, the run itself -- and a project is only asked to
# rename the key once.
_warned_deprecated_deps_key = False
def get_env_input_paths(env: dict, env_name: str | None = None) -> list[str]:
"""Read the files an environment declares it depends on.
Written as ``inputs``, and also accepted as ``deps``, which is the name
this was published under on Docker environments. Extra keys on an
environment are ignored rather than refused, so a project still
spelling it the old way would otherwise have its files go silently
untracked---which is the one failure the field exists to prevent.
Setting both is refused rather than resolved one way, since it is the
same list written twice in two places that can drift apart.
This is the only reader of either spelling, so the alias lives in one
place: the models accept it through ``AliasChoices``, and everything
reading a raw environment dict comes through here.
"""
global _warned_deprecated_deps_key
inputs = env.get("inputs")
deps = env.get("deps")
if inputs is not None and deps is not None:
where = f" on environment '{env_name}'" if env_name else ""
raise ValueError(
f"Both 'inputs' and 'deps' are set{where}; 'deps' is the old "
"name for the same key, so merge them into 'inputs'"
)
if deps is not None and not _warned_deprecated_deps_key:
from calkit.cli import warn
_warned_deprecated_deps_key = True
# Written for whoever has to act on it rather than raised as a
# UserWarning, whose file-and-line formatting reads as a defect in
# Calkit instead of a line to change in their own project
where = f" on environment '{env_name}'" if env_name else ""
warn(
f"The 'deps' key{where} is deprecated; rename it to 'inputs', "
"which is what it's called now that scheduler and system "
"environments take one too"
)
paths = list(inputs if inputs is not None else deps or [])
# Checked here rather than only on the models: every production caller
# reads a raw environment dict, so a model annotation alone would let
# '../outside.sh' through to DVC. Docker's list is exempt because it
# is the long-published 'deps' under a new name, and tightening it
# would retroactively invalidate existing projects.
if env.get("kind") in ("system", "slurm", "pbs"):
from calkit.provenance import check_project_path
for path in paths:
problem = check_project_path(path)
if problem:
where = f" on environment '{env_name}'" if env_name else ""
raise ValueError(f"Environment input{where}: {problem}")
return paths
def get_system_lock_data(
lock: list[str], system_info: dict | None = None
) -> dict:
"""Read the machine properties a ``system`` environment locks.
Raises if a locked property isn't available on the machine, e.g., a
tool that isn't installed. Recording it as null would quietly claim the
stage is pinned to something it isn't, which is worse than not pinning
it at all.
``system_info`` describes a machine other than this one, for an
environment whose host is somewhere else: what the results depend on is
the machine the stage runs on, so that is what gets pinned.
"""
if system_info is None:
system_info = calkit.get_system_info()
data = {}
for prop in lock:
key = SYSTEM_LOCK_PROPERTIES.get(prop)
if key is None:
raise ValueError(
f"Unknown system property to lock: '{prop}'; valid options "
f"are {', '.join(sorted(SYSTEM_LOCK_PROPERTIES))}"
)
value = system_info.get(key)
if value is None:
hint = ""
if prop == "machine-id":
# The one lockable property that can be supplied by hand,
# and the one most likely to be locked implicitly -- so the
# way out is worth naming rather than leaving to the docs
hint = (
"; no identifier could be read from the platform, so "
"give it one with 'calkit config set machine_id <id>'"
)
raise ValueError(
f"System property '{prop}' is not available on this machine"
+ hint
)
data[prop] = SYSTEM_LOCK_PROPERTY_PRECISION.get(prop, lambda v: v)(
value
)
return data
def write_system_env_lock(
env_name: str,
env: dict,
wdir: str | None = None,
system_info: dict | None = None,
) -> str | None:
"""Write a JSON lock file for a ``system`` environment.
Unlike the other lock files, this one describes the machine rather than
a spec the project controls, so it changes when the project moves to a
different machine. That is the intent: a stage that declared it depends
on, say, the Julia version should not reuse a cached result from a box
with a different one.
A non-default ``shell`` is recorded alongside the machine properties.
It isn't a property of the machine, but it feeds the same question: a
stage whose setup commands ran under a different shell should not
reuse the old result. The setup commands themselves aren't here --
they are compiled into the stage's command, where DVC already watches
them.
Returns the lock file path, already prefixed with ``wdir`` if provided,
or None if the environment locks nothing.
"""
# Already prefixed with wdir, so it must not be joined with it again
lock_fpath = get_env_lock_fpath(
env=env, env_name=env_name, wdir=wdir, as_posix=True
)
if lock_fpath is None:
return None
# Read before anything is created: a misspelled property or a tool
# that isn't installed raises here, and an env that failed to lock
# shouldn't leave a directory behind suggesting it did
lock_data = get_system_lock_data(
env.get("lock") or [], system_info=system_info
)
# Named so it can't collide with a locked property, now or when the set
# of them grows. The setup commands themselves are not recorded: they
# go into the stage's command when the pipeline is compiled, so DVC
# sees them change. The shell they run in doesn't, so it does -- and
# only when it isn't the default, since writing the default says
# nothing that leaving it out doesn't.
shell = env.get("shell", SYSTEM_ENV_DEFAULT_SHELL)
if shell != SYSTEM_ENV_DEFAULT_SHELL:
lock_data["shell"] = shell
content = json.dumps(lock_data, indent=2, sort_keys=True) + "\n"
parent = os.path.dirname(lock_fpath)
if parent:
os.makedirs(parent, exist_ok=True)
if os.path.isfile(lock_fpath):
with open(lock_fpath, "r") as f:
if f.read() == content:
return lock_fpath
with open(lock_fpath, "w") as f:
f.write(content)
return lock_fpath
def get_cache_db(name="cache") -> SqliteDict:
env_check_cache_dir = os.path.join(
os.path.expanduser("~"), ".calkit", "env-checks"
)
os.makedirs(env_check_cache_dir, exist_ok=True)
env_check_cache_path = os.path.join(env_check_cache_dir, f"{name}.sqlite")
return SqliteDict(env_check_cache_path)
def make_cache_key(env_name: str, wdir: str | None = None) -> str:
if wdir is None:
wdir = os.getcwd()
else:
wdir = os.path.abspath(wdir)
return f"{wdir}::{env_name}"
def hash_dict(d: dict) -> str:
json_str = json.dumps(d, sort_keys=True)
return hashlib.sha256(json_str.encode()).hexdigest()
def calc_data_for_env(
env_name: str, env: dict, wdir: str | None = None
) -> dict:
"""Hash important data from the environment.
This includes:
1. A hash of the env definition.
2. A hash of the env path file, if present.
3. A hash of the env prefix, if applicable.
4. A hash of the env lock file, if applicable.
"""
def get_cached_md5(path: str) -> str | None:
"""Get a cached MD5 hash for a path.
For files, use mtime as a fast invalidation check. For directories,
use a lightweight directory signature, since directory mtimes can be
unreliable across filesystems and operations.
"""
key = os.path.abspath(path)
cached_data = {}
with get_cache_db(name="md5s") as db:
if key in db:
cached_data = db[key]
if not os.path.exists(path):
return None
if os.path.isdir(path):
# Avoid deep recursive walks on every cache check. A shallow
# signature is enough to decide whether to recompute full MD5.
shallow_sig = _calc_dir_sig_shallow(path, max_depth=2)
if shallow_sig == cached_data.get("shallow_sig"):
return cached_data.get("md5")
md5 = calkit.get_md5(path)
with get_cache_db(name="md5s") as db:
db[key] = {"md5": md5, "shallow_sig": shallow_sig}
db.commit()
return md5
mtime = os.path.getmtime(path)
if mtime == cached_data.get("mtime"):
return cached_data.get("md5")
if os.path.exists(path):
md5 = calkit.get_md5(path)
with get_cache_db(name="md5s") as db:
db[key] = {"md5": md5, "mtime": mtime}
db.commit()
return md5
if wdir is None:
wdir = os.getcwd()
else:
wdir = os.path.abspath(wdir)
env_hash = hash_dict(env)
env_path = env.get("path", "")
env_path_hash = None
if env_path:
env_path_full = os.path.join(wdir, env_path)
if os.path.isfile(env_path_full):
env_path_hash = calkit.get_md5(env_path_full)
env_prefix = env.get("prefix", "")
env_prefix_hash = None
if env_prefix:
env_prefix_full = os.path.join(wdir, env_prefix)
if os.path.exists(env_prefix_full):
env_prefix_hash = get_cached_md5(env_prefix_full)
else:
env_prefix_hash = None
julia_packages_sig = None
if env.get("kind") == "julia":
julia_packages_sig = calc_julia_depot_sig()
env_lock_hash = None
env_lock_fpath = get_env_lock_fpath(env_name=env_name, env=env, wdir=wdir)
if env_lock_fpath is not None:
env_lock_full = os.path.join(wdir, env_lock_fpath)
if os.path.isfile(env_lock_full):
env_lock_hash = calkit.get_md5(env_lock_full)
return {
"hashes": {
"env_hash": env_hash,
"env_path_hash": env_path_hash,
"env_prefix_hash": env_prefix_hash,
"julia_packages_sig": julia_packages_sig,
"env_lock_hash": env_lock_hash,
},
"checked_at": calkit.utcnow(),
}
def check_cache(
env_name: str,
env: dict,
wdir: str | None = None,
respect_ttl: bool = True,
) -> bool:
"""Check if the environment is up-to-date based on cached data."""
if wdir is None:
wdir = os.getcwd()
else:
wdir = os.path.abspath(wdir)
with get_cache_db() as db:
key = make_cache_key(env_name=env_name, wdir=wdir)
if key not in db:
return False
cached_data = db[key]
# If our last check failed, we're definitely not up-to-date
if not cached_data.get("success", False):
return False
if respect_ttl:
cached_checked_at = cached_data.get("checked_at")
if cached_checked_at is None:
return False
time_diff = calkit.utcnow() - cached_checked_at
if time_diff.total_seconds() > ENV_CHECK_CACHE_TTL_SECONDS:
return False
# A Docker environment's image can be deleted without anything the cache
# hashes changing, and the pipeline can't run in an image that isn't
# there, so its absence has to invalidate the check
if env.get("kind") == "docker":
from calkit.docker import get_image_name, image_exists
image = get_image_name(env, env_name, wdir=wdir)
if image and not image_exists(image):
return False
# Check if this environment is up-to-date
current_data = calc_data_for_env(env_name=env_name, env=env, wdir=wdir)
if env.get("path") and not current_data["hashes"]["env_path_hash"]:
return False
if env.get("prefix") and not current_data["hashes"]["env_prefix_hash"]:
return False
if (
get_env_lock_fpath(env=env, env_name=env_name, wdir=wdir)
and not current_data["hashes"]["env_lock_hash"]
):
return False
return current_data["hashes"] == cached_data["hashes"]
def save_cache(
env_name: str, env: dict, wdir: str | None = None, success: bool = True
) -> dict:
with get_cache_db() as db:
key = make_cache_key(env_name=env_name, wdir=wdir)
data = calc_data_for_env(env_name=env_name, env=env, wdir=wdir)
data["success"] = success
db[key] = data
db.commit()
return data
def check_all_in_pipeline(
ck_info: dict | None = None,
wdir: str | None = None,
targets: list[str] | None = None,
force: bool = False,
) -> dict:
"""Check all environments in the pipeline, caching for efficiency.
The cache file is a simple JSON file keyed by project path.
Each object inside tracks the last check timestamp, pass/fail,
and some sort of hash(es) for the important file content involved.
"""
import calkit
from calkit.cli.check import check_environment
# TODO: ``check_environment`` should be able to take a wdir argument
if wdir is not None:
raise ValueError(
"Can currently only run from current working directory"
)
res = {}
# First get a list of environments used in the pipeline
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
# Markdown stages carry no environment of their own; the stages their
# blocks declare do, so expand before looking for environments to check.
import calkit.markdown
md_stages = calkit.markdown.get_markdown_stages(ck_info)
ck_info = calkit.markdown.expand_ck_info(ck_info).ck_info
stages = ck_info.get("pipeline", {}).get("stages", {})
if targets:
# Split targets by "@" to handle sub-stages from iterations
targets = [t.split("@")[0] for t in targets]
# A target naming a markdown stage covers every stage its file
# declares, which are named '<stage>/<block>'
prefixes = [
t + calkit.markdown.STAGE_NAME_SEPARATOR
for t in targets
if t in md_stages
]
stages = {
k: v
for k, v in stages.items()
if k in targets or any(k.startswith(p) for p in prefixes)
}
envs_in_pipeline = [stage.get("environment") for stage in stages.values()]
envs_in_pipeline = [
e for e in envs_in_pipeline if e and not (str(e)).startswith("_")
]
# If any environments are composite environments, we need to split them
# up into their individual names in the list
split_envs = []
for env_name in envs_in_pipeline:
if env_name.count(COMPOSITE_ENV_SEP) == 1:
outer_env_name, sub_env_name = env_name.split(COMPOSITE_ENV_SEP)
split_envs += [outer_env_name, sub_env_name]
else:
split_envs.append(env_name)
envs_in_pipeline = list(set(split_envs))
envs = ck_info.get("environments", {})
for env_name in envs_in_pipeline:
env = envs.get(env_name)
if env.get("kind") in KINDS_NO_CHECK:
continue
if not force:
up_to_date = cacheable(env) and check_cache(
env_name=env_name, env=env, wdir=wdir
)
if up_to_date:
res[env_name] = {"success": True, "cached": True}
continue
try:
check_environment(env_name, verbose=False)
res[env_name] = save_cache(
env_name=env_name, env=env, wdir=wdir, success=True
)
except Exception:
res[env_name] = save_cache(
env_name=env_name, env=env, wdir=wdir, success=False
)
return res
class EnvDetectResult(BaseModel):
name: str
env: dict
exists: bool
outer: "EnvDetectResult | None" = None
class EnvForStageResult(BaseModel):
"""Result of detecting or creating an environment for a stage."""
name: str
env: dict
exists: bool
spec_path: str | None = None
spec_content: str | None = None
dependencies: list[str] = []
created_from_dependencies: bool = False
def make_env_name(path: str, all_env_names: list[str], kind: str) -> str:
"""Generate a unique environment name based on path, existing
names, and kind.
Parameters
----------
path : str
Path to the environment spec file.
all_env_names : list[str]
List of existing environment names.
kind : str
Environment kind (e.g., "uv-venv", "conda", "renv", "julia").
Returns
-------
str
A unique environment name.
"""
dirname = Path(path).parent.name
# If this is the first env in the project, call it main
if not all_env_names:
return dirname or "main"
# Name based on dirname if possible
if dirname and dirname not in all_env_names:
return dirname
# Try a name based on the dirname and kind
if dirname and dirname in all_env_names:
name = f"{dirname}-{kind}"
if name not in all_env_names:
return name
# Otherwise increment a number after the kind
n = 1
name = f"{kind}{n}"
while name in all_env_names:
n += 1
name = f"{kind}{n}"
return name
def get_default_venv_prefix(envs: dict, path: str, name: str) -> str:
"""Return the default prefix for a venv or uv-venv environment.
The prefix defaults to ``.venv`` in the same directory as ``path``,
unless that location is already claimed by another environment, in which
case the virtualenv is nested under ``.calkit/envs/{name}/.venv``. This
is resolved on the fly so that the prefix need not be stored in
``calkit.yaml``.
A location is considered claimed by another environment if it pins that
explicit ``prefix``, or if it is a ``uv``, ``venv``, or ``uv-venv``
environment whose ``.venv`` would live there (``uv`` always creates its
virtualenv at ``.venv`` in its project directory, so a flexible venv
yields to it). Sibling venvs that would otherwise collide all nest under
their own name-scoped location, which is collision-free.
Parameters
----------
envs : dict
All environments, keyed by name.
path : str
Path to the spec file the environment lives alongside.
name : str
Name of the environment being resolved, used both to exclude it from
the claimed locations and for the nested fallback.
Returns
-------
str
A POSIX-style prefix that does not collide with another environment.
"""
base = os.path.join(os.path.dirname(path), ".venv")
# Collect .venv locations claimed by the other environments
claimed = set()
for other_name, env in envs.items():
if other_name == name:
continue
prefix = env.get("prefix")
if prefix is not None:
claimed.add(os.path.normpath(prefix))
elif env.get("kind") in ("uv", "venv", "uv-venv"):
other_dir = os.path.dirname(env.get("path", ""))
claimed.add(os.path.normpath(os.path.join(other_dir, ".venv")))
# Nest under .calkit/envs/{name} if the default location is taken
if os.path.normpath(base) in claimed:
base = os.path.join(".calkit", "envs", name, ".venv")
return Path(base).as_posix()
def env_from_name_or_path(
name_or_path: str | None = None,
ck_info: dict | None = None,
path_only: bool = False,
language: str | None = None,
) -> EnvDetectResult:
"""Get an environment from its name or path.
Names take precedence.
Parameters
----------
name_or_path : str | None
Name or path of the environment. If None and language is provided,
will search for or create a docker environment for that language.
ck_info : dict | None
Calkit info dict. If None, will be loaded from calkit.yaml.
path_only : bool
Only match on path, not name.
language : str | None
Language/tool to detect docker environment for (e.g., "latex").
Only used if name_or_path is None.
Returns
-------
EnvDetectResult
Environment detection result.
"""
# Load config and environment list
if ck_info is None:
ck_info = calkit.load_calkit_info()
envs = ck_info.get("environments", {})
all_env_names = list(envs.keys())
# Handle language-based environment detection
# This will usually use a spec path, not a name in ck_info
if name_or_path is None and language is not None:
# Look for a docker environment matching the language
for env_name, env in envs.items():
if env.get("kind") == "docker":
image = env.get("image", "").lower()
# Check if this looks like a language environment
if language.lower() in image or f"{language}mk" in image:
return EnvDetectResult(name=env_name, env=env, exists=True)
# Only create default docker environment for latex
if language.lower() == "latex":
env_name = "latex"
return EnvDetectResult(
name=env_name,
env={
"kind": "docker",
"image": "texlive/texlive:latest-full",
},
exists=False,
)
# For shell language, use _system environment
if language.lower() == "shell":
return EnvDetectResult(
name="_system",
env={"kind": "system"},
exists=True,
)
# For other languages, try to detect a default environment
default_env = detect_default_env(ck_info=ck_info, language=language)
if default_env:
return default_env
raise ValueError(
f"Could not find or create environment for language: {language}"
)
# Require either name_or_path or language
if name_or_path is None:
raise ValueError("Either name_or_path or language must be provided")
# Check if environment exists by name or path
for env_name, env in envs.items():
if (not path_only and env_name == name_or_path) or env.get(
"path"
) == name_or_path:
return EnvDetectResult(name=env_name, env=env, exists=True)
# Check for nested environments like mycluster:mypython
if name_or_path.count(COMPOSITE_ENV_SEP) == 1 and not path_only:
outer_env_name, sub_env_name = name_or_path.split(COMPOSITE_ENV_SEP)
outer_env = envs.get(outer_env_name)
if outer_env and outer_env.get("kind") in VALID_OUTER_ENV_KINDS:
# Look for an inner environment with the given name and path
for sub_name, sub_env in envs.items():
if (not path_only and sub_name == sub_env_name) or sub_env.get(
"path"
) == sub_env_name:
return EnvDetectResult(
name=sub_name,
env=sub_env,
exists=True,
outer=EnvDetectResult(
name=outer_env_name, env=outer_env, exists=True
),
)
# Handle special _system environment
if name_or_path == "_system":
return EnvDetectResult(
name="_system",
env={"kind": "system"},
exists=True,
)
# Check if name_or_path is a file and detect environment type
env_path = _as_posix_path(name_or_path)
if os.path.isfile(env_path):
if env_path.endswith("requirements.txt"):
# TODO: Detect if uv is installed, and use a plain venv if not
# The prefix is left unset and resolved on the fly at check/run
# time via get_default_venv_prefix
return EnvDetectResult(
name=make_env_name(env_path, all_env_names, kind="uv-venv"),
env={
"kind": "uv-venv",
"path": env_path,
"python": DEFAULT_PYTHON_VERSION,
},
exists=False,
)
elif env_path.endswith(".yml") or env_path.endswith(".yaml"):
# This is probably a Conda env
with open(env_path) as f:
env_spec = calkit.ryaml.load(f)
if "dependencies" not in env_spec:
raise ValueError(
f"Could not detect environment from: {name_or_path}"
)
return EnvDetectResult(
name=env_spec.get(
"name",
make_env_name(env_path, all_env_names, kind="conda"),
),
env={"kind": "conda", "path": env_path},
exists=False,
)
elif env_path.endswith("pyproject.toml"):
# This is a uv project env
return EnvDetectResult(
name=make_env_name(env_path, all_env_names, kind="uv"),
env={
"kind": "uv",
"path": env_path,
},
exists=False,
)
elif env_path.endswith("pixi.toml"):
# This is a pixi env
return EnvDetectResult(
name=make_env_name(env_path, all_env_names, kind="pixi"),
env={
"kind": "pixi",
"path": env_path,
},
exists=False,
)
elif env_path.endswith("Project.toml"):
# This is a Julia env
return EnvDetectResult(
name=make_env_name(env_path, all_env_names, kind="julia"),
env={
"kind": "julia",
"path": env_path,
"julia": _get_julia_version(),
},
exists=False,
)
elif env_path.endswith("DESCRIPTION"):
# This is an R renv environment
return EnvDetectResult(
name=make_env_name(env_path, all_env_names, kind="renv"),
env={"kind": "renv", "path": env_path},
exists=False,
)
elif env_path.endswith("flake.nix"):
# This is a Nix flake environment
return EnvDetectResult(
name=make_env_name(env_path, all_env_names, kind="nix"),
env={"kind": "nix", "path": env_path},
exists=False,
)
elif "dockerfile" in env_path.lower():
# This is a Docker env
project_name = calkit.detect_project_name(prepend_owner=False)
env_name = make_env_name(env_path, all_env_names, kind="docker")
image_name = f"{project_name}-{env_name}"
return EnvDetectResult(
name=env_name,
env={
"kind": "docker",
"path": env_path,
"image": image_name,
},
exists=False,
)
raise ValueError(f"Environment could not be detected from: {name_or_path}")
def env_from_name_and_or_path(
name: str | None, path: str | None, ck_info: dict | None = None
) -> EnvDetectResult:
"""Detect an environment from its name and/or path."""
if ck_info is None:
ck_info = calkit.load_calkit_info()
envs = ck_info.get("environments", {})
path = _as_posix_path(path) if path else None
if name and name in envs:
env = envs[name]
if path and _as_posix_path(env.get("path", "")) != path:
raise ValueError(
f"Environment '{name}' exists but has a different path "
f"('{env.get('path')}') than provided ('{path}')"
)
return EnvDetectResult(name=name, env=envs[name], exists=True)
# Detect composite environments
if name and name.count(COMPOSITE_ENV_SEP) == 1:
outer_env_name, sub_env_name = name.split(COMPOSITE_ENV_SEP)
outer_env = envs.get(outer_env_name)
if outer_env and outer_env.get("kind") in VALID_OUTER_ENV_KINDS:
# Look for a sub-environment with the given name and path
for sub_name, sub_env in envs.items():
if (sub_name == sub_env_name) or (
path and _as_posix_path(sub_env.get("path", "")) == path
):
return EnvDetectResult(
name=sub_name,
env=sub_env,
exists=True,
outer=EnvDetectResult(
name=outer_env_name, env=outer_env, exists=True
),
)
if path:
res = env_from_name_or_path(
name_or_path=path, ck_info=ck_info, path_only=True
)
if name:
res.name = name
return res
# If we have neither name nor path, we can only detect the environment
# if there's only one
default = detect_default_env(ck_info=ck_info)
if default:
return default
raise ValueError(
f"Environment could not be detected from name: {name} "
f"and/or path: {path}"
)
def env_from_notebook_path(
notebook_path: str, ck_info: dict | None = None
) -> EnvDetectResult:
"""Detect an environment for a notebook based on its path.
First we look in pipeline stages, then in the notebooks list.
"""
if ck_info is None:
ck_info = calkit.load_calkit_info()
notebook_path = _as_posix_path(notebook_path)
stages = ck_info.get("pipeline", {}).get("stages", {})
envs = ck_info.get("environments", {})
for stage in stages.values():
if (
stage.get("kind") == "jupyter-notebook"
and _as_posix_path(stage.get("notebook_path", "")) == notebook_path
):
env_name = stage.get("environment")
if env_name:
env = envs.get(env_name)
if env:
return EnvDetectResult(name=env_name, env=env, exists=True)
for nb in ck_info.get("notebooks", []):
if _as_posix_path(nb.get("path", "")) == notebook_path:
env_name = nb.get("environment")
if env_name:
env = envs.get(env_name)
if env:
return EnvDetectResult(name=env_name, env=env, exists=True)
# Fall back to default env if possible
default = detect_default_env(ck_info=ck_info)
if default:
return default
raise ValueError(
f"Environment could not be detected for notebook path: {notebook_path}"
)
def detect_default_env(
ck_info: dict | None = None, language: str | None = None
) -> EnvDetectResult | None:
"""Detect a default environment for the project.
First, if the project has a single environment, we use that. Otherwise,
we look for a single typical env spec file.
Parameters
----------
ck_info : dict | None
Calkit info dict. If None, will be loaded from calkit.yaml.
language : str | None
Language to filter environments by when multiple environments exist.
"""
if ck_info is None:
ck_info = calkit.load_calkit_info()
envs = ck_info.get("environments", {})
if len(envs) == 1:
env_name, env = next(iter(envs.items()))
return EnvDetectResult(name=env_name, env=env, exists=True)
elif len(envs) > 1:
return
# Look for typical env spec files in order
# There must only be one, however, otherwise the default is ambiguous
# Filter by language if provided
if language:
language_lower = language.lower()
if language_lower == "python":
env_spec_paths = [
"pyproject.toml",
"requirements.txt",
"environment.yml",
"pixi.toml",
]
elif language_lower == "julia":
env_spec_paths = ["Project.toml"]
elif language_lower == "r":
env_spec_paths = ["DESCRIPTION", "environment.yml", "pixi.toml"]
elif language_lower == "shell":
env_spec_paths = ["Dockerfile"]
elif language_lower == "matlab":
env_spec_paths = ["Dockerfile"]
else:
# For other languages, use generic list
env_spec_paths = [
"pyproject.toml",
"requirements.txt",
"environment.yml",
"Dockerfile",
"Project.toml",
"renv.lock",
"pixi.toml",
]
else:
# No language specified, use generic list
env_spec_paths = [
"pyproject.toml",
"requirements.txt",
"environment.yml",
"Dockerfile",
"Project.toml",
"DESCRIPTION",
"pixi.toml",
"flake.nix",
]
present = os.listdir(".")
present_env_specs = [p for p in env_spec_paths if p in present]
if len(present_env_specs) == 1:
return env_from_name_or_path(
present_env_specs[0], ck_info=ck_info, path_only=True
)
def create_nix_flake_content(
packages: list[str],
description: str | None = None,
nixpkgs_url: str = "github:NixOS/nixpkgs/nixos-unstable",
) -> str:
"""Generate a minimal flake.nix exposing a default dev shell.
The flake builds a ``devShells.default`` containing the requested
packages from nixpkgs, available on the common Linux + macOS systems.
Reproducibility comes from ``flake.lock``, which pins the nixpkgs
revision and is generated by ``nix flake lock`` after writing this
file.
"""
pkgs_block = "\n".join(f" {p}" for p in packages)
desc = description or "Calkit-managed Nix dev environment"
desc_escaped = desc.replace('"', '\\"')
return f"""{{
description = "{desc_escaped}";
inputs.nixpkgs.url = "{nixpkgs_url}";
outputs = {{ self, nixpkgs }}:
let
systems = [
"x86_64-linux"
"aarch64-linux"
"x86_64-darwin"
"aarch64-darwin"
];
forAllSystems = f:
nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${{system}});
in {{
devShells = forAllSystems (pkgs: {{
default = pkgs.mkShell {{
packages = with pkgs; [
{pkgs_block}
];
}};
}});
}};
}}
"""
def add_packages_to_nix_flake(
flake_path: str, packages: list[str]
) -> list[str]:
"""Add ``packages`` to a flake.nix's ``packages = with pkgs; [ ... ]``
list, preserving formatting and skipping packages already present.
Designed for flakes produced by ``calkit new nix-env``. If the anchor
can't be found (e.g. the user hand-rolled a structurally different
flake), raises ``ValueError`` so the caller can tell the user to edit
manually rather than corrupting their file.
Returns the list of packages actually inserted.
"""
import re
with open(flake_path) as f:
lines = f.readlines()
anchor_re = re.compile(r"packages\s*=\s*with\s+pkgs\s*;\s*\[")
close_re = re.compile(r"^\s*\]\s*;")
start = next(
(i for i, line in enumerate(lines) if anchor_re.search(line)), None
)
if start is None:
raise ValueError(
f"Could not find 'packages = with pkgs; [' in {flake_path}; "
"add packages manually."
)
end = next(
(j for j in range(start + 1, len(lines)) if close_re.match(lines[j])),
None,
)
if end is None:
raise ValueError(
f"Could not find closing ']' for packages list in {flake_path}."
)
# Collect existing entries (ignore blanks + comments) and pick up the
# indent from the first real entry so inserted lines match.
existing: set[str] = set()
inner_indent: str | None = None
for k in range(start + 1, end):
stripped = lines[k].strip()
if not stripped or stripped.startswith("#"):
continue
existing.add(stripped)
if inner_indent is None:
indent_match = re.match(r"^(\s*)", lines[k])
inner_indent = indent_match.group(1) if indent_match else ""
if inner_indent is None:
# Empty list -- derive indent from the closing bracket + 2 spaces.
close_indent_match = re.match(r"^(\s*)", lines[end])
close_indent = (
close_indent_match.group(1) if close_indent_match else ""
)
inner_indent = close_indent + " "
added: list[str] = []
new_entries = []
for pkg in packages:
if pkg in existing:
continue
new_entries.append(f"{inner_indent}{pkg}\n")
existing.add(pkg)
added.append(pkg)
if not new_entries:
return added
lines[end:end] = new_entries
with open(flake_path, "w") as f:
f.writelines(lines)
return added
def create_python_requirements_content(dependencies: list[str]) -> str:
"""Generate requirements.txt file content from a list of dependencies.
Parameters
----------
dependencies : list[str]
List of package names.
Returns
-------
str
The requirements.txt file content.
"""
return "\n".join(dependencies) if dependencies else ""
def create_uv_pyproject_content(
dependencies: list[str],
project_name: str | None = None,
python_version: str = DEFAULT_PYTHON_VERSION,
) -> str:
"""Generate a minimal pyproject.toml for a uv environment.
Parameters
----------
dependencies : list[str]
List of package names.
project_name : str | None
Name of the project. If None, uses the detected project name.
python_version : str
Python version to include in requires-python.
Returns
-------
str
The pyproject.toml file content.
"""
if project_name is None:
project_name = calkit.detect_project_name(prepend_owner=False)
content = "[project]\n"
content += f'name = "{project_name}"\n'
content += 'version = "0.1.0"\n'
content += f'requires-python = ">={python_version}"\n'
if dependencies:
content += "dependencies = [\n"
for dep in sorted(dependencies):
content += f' "{dep}",\n'
content += "]\n"
return content
def _resolve_julia_package_uuids(
package_names: list[str],
) -> dict[str, str]:
"""Resolve Julia package names to their UUIDs using Pkg registry.
Parameters
----------
package_names : list[str]
List of Julia package names to resolve.
Returns
-------
dict[str, str]
Dictionary mapping package names to their UUIDs.
If a package UUID cannot be resolved, it is omitted.
"""
if not package_names:
return {}
# Create Julia script to query Pkg registry for UUIDs
# This safely handles packages that don't exist
julia_code = """
using Pkg
using Pkg.Registry
packages = split(ARGS[1], ",")
registries = Pkg.Registry.reachable_registries()
if isempty(registries)
Pkg.Registry.add("General")
registries = Pkg.Registry.reachable_registries()
end
for pkg in packages
for reg in registries
found = false
# Scan the registry's own package table rather than calling a
# lookup helper; Pkg.Registry.find was removed in Julia 1.12, and
# calling it failed for every package, silently.
for (uuid, entry) in reg.pkgs
if entry.name == pkg
println(pkg * "=" * string(uuid))
found = true
break
end
end
found && break
end
end
"""
try:
# Write Julia script to temp file since passing long code via
# command line can be problematic
with tempfile.NamedTemporaryFile(
mode="w",
suffix=".jl",
delete=False,
) as f:
f.write(julia_code)
script_path = f.name
# Run Julia with the script
result = subprocess.run(
[
calkit.julia.get_julia_exe(),
script_path,
",".join(package_names),
],
capture_output=True,
text=True,
timeout=60,
)
# Clean up temp file
try:
os.unlink(script_path)
except FileNotFoundError:
pass
if result.returncode != 0:
# If Julia fails, return empty dict to fall back
return {}
# Parse output: each line is "package=uuid"
uuids = {}
for line in result.stdout.strip().split("\n"):
if "=" in line:
parts = line.strip().split("=", 1)
if len(parts) == 2:
pkg, uuid = parts
uuids[pkg.strip()] = uuid.strip()
return uuids
except (subprocess.TimeoutExpired, FileNotFoundError, Exception):
# If Julia is not available or times out, return empty dict
return {}
def create_conda_environment_content(
dependencies: list[str],
project_name: str | None = None,
python_version: str = DEFAULT_PYTHON_VERSION,
) -> str:
"""Generate a minimal environment.yml for a conda environment.
Parameters
----------
dependencies : list[str]
List of package names.
project_name : str | None
Name of the environment. If None, uses the detected project name.
python_version : str
Python version to request.
Returns
-------
str
The environment.yml file content.
"""
if project_name is None:
project_name = calkit.detect_project_name(prepend_owner=False)
content = f"name: {project_name}\n"
content += "channels:\n - conda-forge\n"
content += "dependencies:\n"
content += f" - python={python_version}\n"
for dep in sorted(dependencies):
if dep.lower().startswith("python") and dep[6:7] in (
"",
"=",
">",
"<",
):
continue
content += f" - {dep}\n"
return content
def create_julia_project_file_content(
dependencies: list[str],
project_name: str = "environment",
) -> str:
"""Generate Julia Project.toml file content from a list of dependencies.
Parameters
----------
dependencies : list[str]
List of package names.
project_name : str
Name of the Julia project.
Returns
-------
str
The Project.toml file content with [deps] section populated
with UUIDs if Julia is available. Otherwise, includes package
names in comments.
"""
content = f'name = "{project_name}"\n'
version = "0.1.0"
content += f'version = "{version}"\n\n'
if not dependencies:
return content
# Try to resolve UUIDs using Julia's Pkg registry
uuids = _resolve_julia_package_uuids(dependencies)
if uuids:
# We have UUIDs, create proper [deps] section
content += "[deps]\n"
for pkg in sorted(dependencies):
if pkg in uuids:
content += f'{pkg} = "{uuids[pkg]}"\n'
return content
else:
# Fallback: Julia not available or registry lookup failed
# Include package names in comments for manual addition
content += "[deps]\n"
content += "# Dependencies (add with Julia's Pkg.add):\n"
content += "# " + ", ".join(sorted(dependencies)) + "\n"
return content
def create_r_description_content(
dependencies: list[str], project_name: str | None = None
) -> str:
"""Generate R DESCRIPTION file content listing dependencies.
This creates a minimal DESCRIPTION file that renv can work with.
Parameters
----------
dependencies : list[str]
List of R package names.
project_name : str | None
Name for the DESCRIPTION's ``Package`` field. It is sanitized into a
valid R package name (letters, numbers and dots, starting with a
letter). Falls back to the detected project name, then to
``CalkitProject``.
Returns
-------
str
The DESCRIPTION file content.
"""
if project_name is None:
project_name = calkit.detect_project_name(prepend_owner=False)
# Valid R package names may only contain letters, numbers and dots, and
# must start with a letter, so replace anything else (e.g. hyphens) with a
# dot and prefix a letter if needed
package_name = re.sub(r"[^A-Za-z0-9.]+", ".", project_name).strip(".")
if not package_name or not package_name[0].isalpha():
package_name = "Calkit." + package_name if package_name else "Calkit"
content = f"""Package: {package_name}
Version: 0.0.1
Title: Auto-generated R environment
"""
if dependencies:
if len(dependencies) == 1:
content += f"Imports: {dependencies[0]}\n"
else:
# Format with first package on same line, rest indented
content += f"Imports: {dependencies[0]},\n"
for i, dep in enumerate(dependencies[1:], 1):
if i < len(dependencies) - 1:
content += f" {dep},\n"
else:
content += f" {dep}\n"
return content
def extract_dependencies_from_spec_file(
spec_path: str, language: str | None = None
) -> list[str]:
"""Extract dependencies from an environment spec file.
Parameters
----------
spec_path : str
Path to the spec file (requirements.txt, Project.toml, etc.).
language : str | None
Language hint to help identify the format. If None, will be inferred
from the file path.
Returns
-------
list[str]
List of package/dependency names.
"""
if not os.path.exists(spec_path):
return []
try:
with open(spec_path, "r", encoding="utf-8") as f:
content = f.read()
except (IOError, UnicodeDecodeError):
return []
# Determine format from filename if not provided
if language is None:
if spec_path.endswith("requirements.txt"):
language = "python-requirements"
elif spec_path.endswith("pyproject.toml"):
language = "python-pyproject"
elif spec_path.endswith("Project.toml"):
language = "julia"
elif spec_path.endswith("DESCRIPTION"):
language = "r"
elif spec_path.endswith("environment.yml"):
language = "conda"
dependencies: list[str] = []
if language in ["python-requirements"]:
# Parse requirements.txt
for line in content.split("\n"):
line = line.strip()
if line and not line.startswith("#"):
# Extract package name (before any version specifiers)
pkg = line.split("[")[0].split("==")[0].split(">=")[0]
pkg = pkg.split("<=")[0].split(">")[0].split("<")[0]
pkg = pkg.split("~=")[0].strip()
if pkg:
dependencies.append(pkg)
elif language == "python-pyproject":
# Parse pyproject.toml to extract dependencies
try:
data = toml.loads(content)
project_deps = data.get("project", {}).get("dependencies", [])
for dep in project_deps:
# Extract package name (before any version specifiers)
pkg = dep.split("[")[0].split("==")[0].split(">=")[0]
pkg = pkg.split("<=")[0].split(">")[0].split("<")[0]
pkg = pkg.split("~=")[0].strip()
if pkg:
dependencies.append(pkg)
except Exception:
pass
elif language == "julia":
# Parse Julia Project.toml for [deps] section
try:
data = toml.loads(content)
# Package names are the keys in the [deps] section
deps_section = data.get("deps", {})
if isinstance(deps_section, dict):
dependencies = list(deps_section.keys())
except Exception:
pass
elif language == "r":
# Parse R DESCRIPTION file for Imports/Depends fields,
# correctly handling multi-line (continued) fields where
# continuation lines start with whitespace.
lines = content.splitlines()
i = 0
while i < len(lines):
raw_line = lines[i]
stripped = raw_line.lstrip()
if stripped.startswith(("Imports:", "Depends:")):
# Extract the text after the field name and colon
_, after_colon = stripped.split(":", 1)
pkg_chunks: list[str] = [after_colon.strip()]
# Collect continuation lines that start with whitespace
j = i + 1
while j < len(lines) and (
lines[j].startswith(" ") or lines[j].startswith("\t")
):
pkg_chunks.append(lines[j].strip())
j += 1
# Join all chunks into a single dependency string
pkg_str = " ".join(pkg_chunks)
# Split on commas to get individual package entries
pkgs = [p.strip() for p in pkg_str.split(",") if p.strip()]
for pkg in pkgs:
# Remove version specifications if present
pkg = pkg.split("(", 1)[0].strip()
if pkg:
dependencies.append(pkg)
# Continue parsing from the first non-continuation line
i = j
continue
i += 1
elif language == "conda":
# Parse conda environment.yml
try:
data = yaml.safe_load(content)
deps = data.get("dependencies", [])
for dep in deps:
if isinstance(dep, str):
# Extract package name (before version spec)
pkg = dep.split("==")[0].split(">=")[0].split("<=")[0]
pkg = pkg.split("=")[0].strip()
if "::" in pkg:
pkg = pkg.split("::", 1)[1]
if pkg:
dependencies.append(pkg)
elif isinstance(dep, dict):
# Handle pip dependencies nested as {pip: [...]}
pip_deps = dep.get("pip", [])
for pip_dep in pip_deps:
if isinstance(pip_dep, str):
pkg = (
pip_dep.split("==")[0]
.split(">=")[0]
.split("<=")[0]
.split("[")[0]
.strip()
)
if "::" in pkg:
pkg = pkg.split("::", 1)[1]
if pkg:
dependencies.append(pkg)
except Exception:
pass
# Remove duplicates and sort
return sorted(list(set(dependencies)))
def env_has_superset_dependencies(
env: dict,
required_deps: list[str],
env_spec_path: str | None = None,
strict: bool = False,
) -> bool:
"""Check if an environment has a superset of required dependencies.
Parameters
----------
env : dict
Environment dict from calkit.yaml with 'kind' and 'path' keys.
required_deps : list[str]
List of required dependencies to check for.
env_spec_path : str | None
Path to the environment spec file. If None, will use env.get("path").
strict : bool
If True, require the spec file to exist and have extractable
dependencies. If False, be optimistic when spec file doesn't exist.
Returns
-------
bool
True if the environment contains all required dependencies,
False otherwise.
"""
if not required_deps:
# No dependencies to check, so any environment works
return True
if env_spec_path is None:
env_spec_path = env.get("path")
if not env_spec_path:
# No path to check
if strict:
return False
return True
if not os.path.exists(env_spec_path):
# Spec file doesn't exist
if strict:
# Strict mode: can't verify, so reject
return False
# Optimistic mode: assume it might work
return True
# Extract dependencies from the environment's spec file
env_deps = extract_dependencies_from_spec_file(env_spec_path)
if not env_deps:
# Couldn't extract dependencies (or file is empty)
if strict:
# Strict mode: can't verify, so reject
return False
# Optimistic mode: assume it might work
return True
# Check if env_deps is a superset of required_deps
# (case-insensitive comparison for package names)
env_deps_lower = {dep.lower() for dep in env_deps}
required_deps_lower = {dep.lower() for dep in required_deps}
transitive = {
"jupyter": {"ipykernel"},
"pandas": {"numpy"},
}
for base, provides in transitive.items():
if base in env_deps_lower:
env_deps_lower.update(provides)
return required_deps_lower.issubset(env_deps_lower)
def detect_env_for_stage(
stage: dict,
environment: str | None = None,
ck_info: dict | None = None,
language: str | None = None,
) -> EnvForStageResult:
"""Detect or create an environment for a pipeline stage.
This function first attempts to detect an existing environment. If that
fails, it detects dependencies from the stage and creates an environment
spec file.
Parameters
----------
stage : dict
The pipeline stage dict with 'kind' and script/notebook paths.
environment : str | None
Optional environment name or path to use. If None, will be detected.
ck_info : dict | None
Calkit info dict. If None, will be loaded from calkit.yaml.
language : str | None
Language hint for environment detection.
Returns
-------
EnvForStageResult
Result containing environment info, spec path, content,
and dependencies.
"""
from calkit.detect import (
detect_dependencies_from_notebook,
detect_julia_dependencies,
detect_python_dependencies,
detect_r_dependencies,
language_from_notebook,
)
if ck_info is None:
ck_info = calkit.load_calkit_info()
# Get existing environment names
envs = ck_info.get("environments", {})
all_env_names = list(envs.keys())
# Generic shell commands should run on the host when no env is set.
if environment is None and stage.get("kind") == "shell-command":
return EnvForStageResult(
name="_system",
env={"kind": "system"},
exists=True,
spec_path=None,
dependencies=[],
created_from_dependencies=False,
)
# 1) If stage has an environment, use that
if environment is not None:
res = env_from_name_or_path(
name_or_path=environment, ck_info=ck_info, language=language
)
return EnvForStageResult(
name=res.name,
env=res.env,
exists=res.exists,
spec_path=res.env.get("path"),
dependencies=[],
created_from_dependencies=False,
)
# Infer stage language if not provided
stage_kind = stage.get("kind")
stage_language = language
if stage_language is None:
if stage_kind == "jupyter-notebook":
stage_language = (
language_from_notebook(stage["notebook_path"]) or "python"
)
elif stage_kind == "python-script":
stage_language = "python"
elif stage_kind == "r-script":
stage_language = "r"
elif stage_kind == "julia-script":
stage_language = "julia"
elif stage_kind == "latex":
stage_language = "latex"
elif stage_kind in ["matlab-script", "matlab-command"]:
stage_language = "matlab"
elif stage_kind in ["shell-script", "shell-command"]:
stage_language = "shell"
language_kinds = {
"python": ["uv", "uv-venv", "venv", "conda", "pixi"],
"r": ["renv", "conda", "pixi"],
"julia": ["julia"],
"matlab": ["matlab"],
"latex": ["docker"],
"shell": ["system"],
}
preferred_kinds = (
language_kinds.get(stage_language, []) if stage_language else []
)
is_first_env_for_language = not any(
env.get("kind") in preferred_kinds for env in envs.values()
)
# Stages with analyzable content where we should check dependencies before
# reusing existing environments
analyzable_stages = {
"jupyter-notebook",
"python-script",
"r-script",
"julia-script",
"shell-script",
}
# Initialize detected_dependencies so it's available throughout function
detected_dependencies: list[str] = []
# For analyzable stages, detect dependencies and check if existing
# environments satisfy them
if stage_language and stage_kind in analyzable_stages:
if stage_kind == "python-script":
detected_dependencies = detect_python_dependencies(
script_path=stage["script_path"]
)
elif stage_kind == "r-script":
detected_dependencies = detect_r_dependencies(
script_path=stage["script_path"]
)
elif stage_kind == "julia-script":
detected_dependencies = detect_julia_dependencies(
script_path=stage["script_path"]
)
elif stage_kind == "jupyter-notebook":
notebook_lang = language_from_notebook(stage["notebook_path"])
detected_dependencies = detect_dependencies_from_notebook(
stage["notebook_path"], language=notebook_lang
)
elif stage_kind == "matlab-script":
# MATLAB detection if needed
detected_dependencies = []
elif stage_kind == "shell-script":
# Shell script detection if needed
detected_dependencies = []
# Check if any existing environment has all these dependencies
matching_envs = [
(name, env)
for name, env in envs.items()
if env.get("kind") in preferred_kinds
]
if matching_envs and detected_dependencies:
# Check if any matching environment has all required dependencies
# Use strict mode: only reuse if we can verify the deps are satisfied
for env_name, env in sorted(
matching_envs, key=lambda item: item[0]
):
if env_has_superset_dependencies(
env, detected_dependencies, strict=True
):
env_name = cast(str, env_name)
return EnvForStageResult(
name=env_name,
env=env,
exists=True,
spec_path=env.get("path"),
dependencies=detected_dependencies,
created_from_dependencies=False,
)
# No existing environment has verified dependencies, fall through to create
# If no matching environment found or no dependencies detected,
# fall through to create one from dependencies
# 2) If there is already an environment for the stage language (for
# non-analyzable stages or analyzable stages with no match), use that
if stage_language and stage_kind not in analyzable_stages:
matching_envs = [
(name, env)
for name, env in envs.items()
if env.get("kind") in preferred_kinds
]
if matching_envs:
env_name, env = sorted(matching_envs, key=lambda item: item[0])[0]
env_name = cast(str, env_name)
return EnvForStageResult(
name=env_name,
env=env,
exists=True,
spec_path=env.get("path"),
dependencies=[],
created_from_dependencies=False,
)
if stage_language == "matlab":
return EnvForStageResult(
name="_system",
env={"kind": "system"},
exists=True,
spec_path=None,
dependencies=[],
created_from_dependencies=False,
)
# 3) If a typical env spec exists for the stage language, use that
# (fallback for analyzable stages if no existing environment matched)
if stage_language:
if stage_language == "latex":
res = env_from_name_or_path(
name_or_path=None,
ck_info=ck_info,
language=stage_language,
)
return EnvForStageResult(
name=res.name,
env=res.env,
exists=res.exists,
spec_path=res.env.get("path"),
dependencies=[],
created_from_dependencies=False,
)
spec_candidates = {
"python": [
"pyproject.toml",
"requirements.txt",
"environment.yml",
"env/*.yml",
"envs/*.yml",
"pixi.toml",
],
"r": [
"DESCRIPTION",
"environment.yml",
"env/*.yml",
"envs/*.yml",
"pixi.toml",
],
"julia": ["Project.toml"],
"shell": ["Dockerfile"],
}
for spec_path in spec_candidates.get(stage_language, []):
if "*" in spec_path:
matches = sorted(glob.glob(spec_path))
if matches:
spec_path = matches[0]
else:
continue
if os.path.isfile(spec_path):
res = env_from_name_or_path(
name_or_path=spec_path,
ck_info=ck_info,
language=stage_language,
)
# For analyzable stages with detected dependencies, verify the
# spec file has all required packages before reusing
if stage_kind in analyzable_stages:
# Detect dependencies for this stage if not already done
if not detected_dependencies:
if stage_kind == "python-script":
detected_dependencies = detect_python_dependencies(
script_path=stage["script_path"]
)
elif stage_kind == "r-script":
detected_dependencies = detect_r_dependencies(
script_path=stage["script_path"]
)
elif stage_kind == "julia-script":
detected_dependencies = detect_julia_dependencies(
script_path=stage["script_path"]
)
elif stage_kind == "jupyter-notebook":
notebook_lang = language_from_notebook(
stage["notebook_path"]
)
detected_dependencies = (
detect_dependencies_from_notebook(
stage["notebook_path"],
language=notebook_lang,
)
)
# Only reuse if it has all the dependencies (strict mode)
if (
detected_dependencies
and not env_has_superset_dependencies(
res.env,
detected_dependencies,
spec_path,
strict=True,
)
):
# This spec file doesn't have all deps, try next
# candidate
continue
return EnvForStageResult(
name=res.name,
env=res.env,
exists=res.exists,
spec_path=res.env.get("path"),
dependencies=[],
created_from_dependencies=False,
)
dependencies: list[str] = []
spec_path: str | None = None
spec_content: str | None = None
env_name: str | None = None
env_dict: dict = {}
# Detect dependencies based on stage kind
if stage["kind"] == "python-script":
dependencies = detect_python_dependencies(
script_path=stage["script_path"]
)
project_name = calkit.detect_project_name(prepend_owner=False)
# Generate unique environment name
if is_first_env_for_language:
temp_path = "pyproject.toml"
env_name = make_env_name(temp_path, all_env_names, kind="uv")
spec_path = "pyproject.toml"
spec_content = create_uv_pyproject_content(dependencies)
env_dict = {
"kind": "uv",
"path": spec_path,
}
else:
temp_path = ".calkit/envs/py/pyproject.toml"
env_name = make_env_name(temp_path, all_env_names, kind="uv")
spec_path = f".calkit/envs/{env_name}/pyproject.toml"
spec_content = create_uv_pyproject_content(
dependencies,
project_name=f"{project_name}-{env_name}",
)
env_dict = {
"kind": "uv",
"path": spec_path,
}
elif stage["kind"] == "r-script":
dependencies = detect_r_dependencies(script_path=stage["script_path"])
# Generate unique environment name
if is_first_env_for_language:
temp_path = "DESCRIPTION"
env_name = make_env_name(temp_path, all_env_names, kind="renv")
spec_path = "DESCRIPTION"
else:
temp_path = ".calkit/envs/r/DESCRIPTION"
env_name = make_env_name(temp_path, all_env_names, kind="renv")
spec_path = f".calkit/envs/{env_name}/DESCRIPTION"
spec_content = create_r_description_content(dependencies)
env_dict = {
"kind": "renv",
"path": spec_path,
}
elif stage["kind"] == "julia-script":
dependencies = detect_julia_dependencies(
script_path=stage["script_path"]
)
project_name = calkit.detect_project_name(prepend_owner=False)
# Generate unique environment name
if is_first_env_for_language:
temp_path = "Project.toml"
env_name = make_env_name(temp_path, all_env_names, kind="julia")
spec_path = "Project.toml"
julia_env_name = project_name
else:
temp_path = ".calkit/envs/julia/Project.toml"
env_name = make_env_name(temp_path, all_env_names, kind="julia")
spec_path = f".calkit/envs/{env_name}/Project.toml"
julia_env_name = f"{project_name}-{env_name}"
spec_content = create_julia_project_file_content(
dependencies, project_name=julia_env_name
)
env_dict = {
"kind": "julia",
"path": spec_path,
"julia": _get_julia_version(),
}
elif stage["kind"] == "jupyter-notebook":
notebook_lang = language_from_notebook(stage["notebook_path"])
dependencies = detect_dependencies_from_notebook(
stage["notebook_path"], language=notebook_lang
)
if notebook_lang == "python" or notebook_lang is None:
project_name = calkit.detect_project_name(prepend_owner=False)
# Add ipykernel for Jupyter notebook support
if "ipykernel" not in dependencies:
dependencies.append("ipykernel")
# Generate unique environment name
if is_first_env_for_language:
temp_path = "pyproject.toml"
env_name = make_env_name(temp_path, all_env_names, kind="uv")
spec_path = "pyproject.toml"
spec_content = create_uv_pyproject_content(dependencies)
env_dict = {
"kind": "uv",
"path": spec_path,
}
else:
temp_path = ".calkit/envs/py/pyproject.toml"
env_name = make_env_name(temp_path, all_env_names, kind="uv")
spec_path = f".calkit/envs/{env_name}/pyproject.toml"
spec_content = create_uv_pyproject_content(
dependencies,
project_name=f"{project_name}-{env_name}",
)
env_dict = {
"kind": "uv",
"path": spec_path,
}
elif notebook_lang == "r":
# Add IRkernel for Jupyter notebook support
if "IRkernel" not in dependencies:
dependencies.append("IRkernel")
# Generate unique environment name
if is_first_env_for_language:
temp_path = "DESCRIPTION"
env_name = make_env_name(temp_path, all_env_names, kind="renv")
spec_path = "DESCRIPTION"
else:
temp_path = ".calkit/envs/r/DESCRIPTION"
env_name = make_env_name(temp_path, all_env_names, kind="renv")
spec_path = f".calkit/envs/{env_name}/DESCRIPTION"
spec_content = create_r_description_content(dependencies)
env_dict = {
"kind": "renv",
"path": spec_path,
}
elif notebook_lang == "julia":
# Add IJulia for Jupyter notebook support
if "IJulia" not in dependencies:
dependencies.append("IJulia")
project_name = calkit.detect_project_name(prepend_owner=False)
# Generate unique environment name
if is_first_env_for_language:
temp_path = "Project.toml"
env_name = make_env_name(
temp_path, all_env_names, kind="julia"
)
spec_path = "Project.toml"
julia_env_name = project_name
else:
temp_path = ".calkit/envs/julia/Project.toml"
env_name = make_env_name(
temp_path, all_env_names, kind="julia"
)
spec_path = f".calkit/envs/{env_name}/Project.toml"
julia_env_name = f"{project_name}-{env_name}"
spec_content = create_julia_project_file_content(
dependencies, project_name=julia_env_name
)
env_dict = {
"kind": "julia",
"path": spec_path,
"julia": _get_julia_version(),
}
if not spec_path or not env_name:
raise ValueError(
f"Could not create environment for stage kind: {stage.get('kind')}"
)
return EnvForStageResult(
name=env_name,
env=env_dict,
exists=False,
spec_path=spec_path,
spec_content=spec_content,
dependencies=dependencies,
created_from_dependencies=True,
)
"""Functionality related to environments."""
import functools
import glob
import hashlib
import json
import os
import platform
import re
import socket
import subprocess
import tempfile
from pathlib import Path
from typing import cast
import toml
import yaml
from pydantic import BaseModel
from sqlitedict import SqliteDict
import calkit
DOCKER_ARCHS = [
"amd64",
"arm64",
"arm-v7",
"arm-v6",
"arm-v5",
"ppc64le",
"s390x",
"386",
"riscv64",
]
# Architectures Docker environments are locked for by default, so that
# moving a project between them doesn't invalidate every stage
DEFAULT_DOCKER_LOCK_ARCHS = ["amd64", "arm64"]
DEFAULT_PYTHON_VERSION = "3.14"
CONDA_VENV_ARCHS = [
"osx-arm64",
"osx-64",
"linux-aarch64",
"linux-ppc64le",
"linux-64",
"win-64",
]
ENV_CHECK_CACHE_TTL_SECONDS = 3600
# Scheduler environment keys that govern how a job is dispatched rather than
# what it computes. They are excluded from the environment lock file, so
# changing them does not invalidate cached results: pacing submissions to be
# polite to a shared queue must not force every simulation to rerun.
SCHEDULER_DISPATCH_ONLY_KEYS = {"max_concurrent_jobs"}
# Environment kinds with nothing to build or verify. Note this matches on
# ``kind``; the special ``_system`` environment is excluded by name instead,
# since names starting with an underscore are filtered out of the pipeline's
# environment list before we get here. ``system`` is not listed: checking one
# means writing its lock file, so it has to go through ``check_environment``.
# No kind is unconditionally uncheckable. A ``system`` env on another host
# comes close, but that depends on the env's host rather than its kind, and
# ``check_environment`` handles it: with nothing locked there is nothing to
# check, and locking a machine we can't observe is an error.
KINDS_NO_CHECK: list[str] = []
# Kinds whose check must not be cached. Caching exists to skip rebuilding
# something expensive, but checking a ``system`` env *is* reading the
# machine -- there is nothing to skip. Caching it means a locked property
# can change and be missed until the cache expires, which is precisely the
# drift ``lock`` exists to catch.
KINDS_NO_CACHE = ["system"]
def cacheable(env: dict) -> bool:
"""Whether an environment's check is worth remembering."""
return env.get("kind") not in KINDS_NO_CACHE
# Maps the kebab-case properties a ``system`` environment can lock onto the
# keys ``get_system_info`` returns. Not a mechanical transformation, hence
# the explicit table: note ``Rscript_version``'s capital R.
SYSTEM_LOCK_PROPERTIES = {
"os": "os",
"os-version": "os_version",
"platform": "platform",
"machine": "machine",
"processor": "processor",
"hostname": "hostname",
"machine-id": "machine_id",
"cpu-count": "cpu_count",
"memory-gb": "memory_gb",
"python-version": "python_version",
"python-implementation": "python_implementation",
"git-version": "git_version",
"docker-version": "docker_version",
"conda-version": "conda_version",
"mamba-version": "mamba_version",
"uv-version": "uv_version",
"pixi-version": "pixi_version",
"julia-version": "julia_version",
"juliaup-version": "juliaup_version",
"rscript-version": "Rscript_version",
"brew-version": "brew_version",
Xet Storage Details
- Size:
- 102 kB
- Xet hash:
- efe722df495db89513dad110fef56593569599e760bb3ae76874869629ed3160
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.