Buckets:

download
raw
44.2 kB
"""Functionality for working with Docker.""" from __future__ import annotations import json import os import re import shlex import subprocess from pathlib import Path from pydantic import BaseModel MINIFORGE_LAYER_TXT = r""" # Install Miniforge ARG MINIFORGE_NAME=Miniforge3 ARG MINIFORGE_VERSION=24.9.2-0 ARG TARGETPLATFORM ENV CONDA_DIR=/opt/conda ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH=${CONDA_DIR}/bin:${PATH} # 1. Install just enough for conda to work # 2. Keep $HOME clean (no .wget-hsts file), since HSTS isn't useful in this context # 3. Install miniforge from GitHub releases # 4. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html # Particularly, we remove pyc and a files. The default install has no js, we can skip that # 5. Activate base by default when running as any *non-root* user as well # Good security practice requires running most workloads as non-root # This makes sure any non-root users created also have base activated # for their interactive shells. # 6. Activate base by default when running as root as well # The root user is already created, so won't pick up changes to /etc/skel RUN apt-get update > /dev/null && \ apt-get install --no-install-recommends --yes \ wget bzip2 ca-certificates \ git \ tini \ > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \ /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \ rm /tmp/miniforge.sh && \ conda clean --tarballs --index-cache --packages --yes && \ find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \ find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \ conda clean --force-pkgs-dirs --all --yes && \ echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc """.strip() FOAMPY_LAYER_TEXT = r""" RUN pip install --no-cache-dir numpy pandas matplotlib h5py \ && pip install --no-cache-dir scipy \ && pip install --no-cache-dir foampy """.strip() UV_LAYER_TEXT = """ COPY --from=ghcr.io/astral-sh/uv:0.8.5 /uv /uvx /bin/ """ JULIA_LAYER_TEXT = """ # Install Julia # Ensure base image is a bullseye distribution COPY --from=julia:1.11.6-bullseye /usr/local/julia /usr/local/julia ENV JULIA_PATH=/usr/local/julia \ PATH=$PATH:/usr/local/julia/bin \ JULIA_GPG=3673DF529D9049477F76B37566E3C7DC03D6E495 \ JULIA_VERSION=1.11.6 """ LAYERS = { "mambaforge": MINIFORGE_LAYER_TXT, "miniforge": MINIFORGE_LAYER_TXT, "foampy": FOAMPY_LAYER_TEXT, "uv": UV_LAYER_TEXT, "julia": JULIA_LAYER_TEXT, } # Docker images whose commands should be passed directly to the image # entrypoint when normalizing `xr` commands XR_DOCKER_ENTRYPOINT_IMAGES = { "minlag/mermaid-cli", } class NormalizedXRDockerCommand(BaseModel): """Normalized Docker command metadata for `calkit xr`.""" image: str wdir: str command: list[str] inputs: list[str] outputs: list[str] environment_name: str stage_name: str | None = None description: str | None = None command_mode: str = "shell" def _image_name_without_tag_or_digest(image: str) -> str: """Return an image name without tag or digest components.""" name = image.split("@", 1)[0] last_slash = name.rfind("/") last_colon = name.rfind(":") # Only strip a tag when the colon appears in the final path segment. if last_colon > last_slash: name = name[:last_colon] return name.lower() def _sanitize_stage_name(stage_name: str) -> str: """Normalize a stage name to the same kebab-case style used by xr.""" stage_name = stage_name.replace("_", "-").lower() stage_name = stage_name.replace(".", "-") stage_name = stage_name.replace(" ", "-") stage_name = re.sub(r"[(){}\[\]'\"><|&;/]", "", stage_name) stage_name = re.sub(r"-+", "-", stage_name) stage_name = stage_name.strip("-") if not stage_name: return "stage" return stage_name def _uses_entrypoint_command_mode(image: str) -> bool: """Return True when image is in the `xr` entrypoint-mode allowlist.""" image_name = _image_name_without_tag_or_digest(image) for configured in XR_DOCKER_ENTRYPOINT_IMAGES: configured_name = configured.lower() if image_name == configured_name: return True if image_name.endswith("/" + configured_name): return True return False def split_xr_command(cmd: list[str]) -> list[str]: """Split a single quoted `docker run ...` command into argv tokens.""" if len(cmd) != 1: return cmd if not cmd[0].lstrip().startswith("docker run"): return cmd try: return shlex.split(cmd[0]) except ValueError: return cmd def _normalize_docker_image(image: str) -> str: """Ensure Docker image references include an explicit tag.""" if "@" in image: return image if ":" in image.rsplit("/", 1)[-1]: return image return image + ":latest" def _parse_volume_spec(volume_spec: str) -> tuple[str, str] | None: """Parse a Docker volume spec into source and destination paths.""" if ":" not in volume_spec: return None parts = volume_spec.rsplit(":", 2) source = "" dest = "" if len(parts) == 2: source, dest = parts elif len(parts) == 3: first, second, third = parts # Handle Windows drive-letter sources with no explicit mode, e.g., # C:\path:/data, which rsplit(':', 2) yields as # ["C", "\\path", "/data"] if ( len(first) == 1 and first.isalpha() and second.startswith(("\\", "/")) ): source, dest = first + ":" + second, third else: # Assume source:dest:mode and ignore the optional mode segment source, dest = first, second else: return None if not source or not dest: return None return source, dest def _to_project_relative_path(path: str, cwd: Path) -> str | None: """Resolve a path and return it relative to the project root when possible.""" path_obj = Path(os.path.expanduser(path)) if not path_obj.is_absolute(): path_obj = (cwd / path_obj).resolve(strict=False) else: path_obj = path_obj.resolve(strict=False) try: return path_obj.relative_to(cwd).as_posix() except ValueError: return None def _parse_docker_run_command(cmd: list[str]) -> dict | None: """Parse a `docker run` argv list into image, args, mounts, and workdir.""" if len(cmd) < 3 or cmd[0] != "docker" or cmd[1] != "run": return None no_arg_opts = {"--rm", "-i", "-t", "-it"} one_arg_opts = { "-u": "user", "--user": "user", "-e": "env", "--env": "env", "--env-file": "env-file", "--network": "network", "--pull": "pull", "-v": "volume", "--volume": "volume", "-w": "workdir", "--workdir": "workdir", "--platform": "platform", "--gpus": "gpus", "-p": "port", "--publish": "port", "--name": "name", "--entrypoint": "entrypoint", } volume_specs: list[str] = [] workdir = None image = None idx = 2 while idx < len(cmd): token = cmd[idx] if token == "--": idx += 1 break if token in no_arg_opts: idx += 1 continue if token in one_arg_opts: if idx + 1 >= len(cmd): return None value = cmd[idx + 1] if one_arg_opts[token] == "volume": volume_specs.append(value) elif one_arg_opts[token] == "workdir": workdir = value idx += 2 continue if token.startswith("--volume="): volume_specs.append(token.split("=", 1)[1]) idx += 1 continue if token.startswith("--workdir="): workdir = token.split("=", 1)[1] idx += 1 continue if token.startswith("--env="): idx += 1 continue if token.startswith("--env-file="): idx += 1 continue if token.startswith("--network="): idx += 1 continue if token.startswith("--pull="): idx += 1 continue if token.startswith("--user="): idx += 1 continue if token.startswith("-"): # Unknown docker option: skip and continue scanning for image. # This keeps normalization resilient to common flags we don't # explicitly model. idx += 1 continue image = token idx += 1 break if image is None: return None return { "image": image, "workdir": workdir, "volumes": volume_specs, "command": cmd[idx:], } def _map_container_path_to_project( path: str, source_prefix: str | None, container_wdir: str, ) -> str: """Map a container path back to a project-relative path when possible.""" path = path.strip() if source_prefix is None: return path source_prefix = source_prefix.strip("/") if path.startswith(container_wdir.rstrip("/") + "/"): suffix = path[len(container_wdir.rstrip("/")) + 1 :] if source_prefix: return Path(source_prefix, suffix).as_posix() return Path(suffix).as_posix() if os.path.isabs(path): return path if source_prefix and path.startswith(source_prefix + "/"): return path if source_prefix: return Path(source_prefix, path).as_posix() return Path(path).as_posix() def normalize_xr_docker_command( cmd: list[str], environment: str | None = None, cwd: str | None = None, ) -> NormalizedXRDockerCommand | None: """Normalize supported `docker run` commands into `xr` command metadata.""" cwd_path = Path(cwd or ".").resolve() cmd = split_xr_command(cmd) parsed = _parse_docker_run_command(cmd) if parsed is None: return None image = _normalize_docker_image(parsed["image"]) if not _uses_entrypoint_command_mode(image): return None image_name = _image_name_without_tag_or_digest(image) is_mermaid_image = image_name.endswith("mermaid-cli") chosen_mount = None volumes = parsed["volumes"] if parsed["workdir"] is not None: for volume_spec in volumes: parsed_volume = _parse_volume_spec(volume_spec) if parsed_volume is None: continue source, dest = parsed_volume if dest == parsed["workdir"]: chosen_mount = (source, dest) break if chosen_mount is None and volumes: parsed_volume = _parse_volume_spec(volumes[0]) if parsed_volume is not None: chosen_mount = parsed_volume container_wdir = parsed["workdir"] if container_wdir is None: if chosen_mount is not None: container_wdir = chosen_mount[1] else: container_wdir = "/data" if is_mermaid_image else "/work" source_prefix = None if chosen_mount is not None: source_prefix = _to_project_relative_path(chosen_mount[0], cwd_path) if source_prefix is None: source_prefix = chosen_mount[0] command_tokens = parsed["command"] if not command_tokens: return None normalized_args: list[str] = list(command_tokens) detected_inputs: list[str] = [] detected_outputs: list[str] = [] if is_mermaid_image: normalized_args = [] idx = 0 while idx < len(command_tokens): token = command_tokens[idx] if token in ["-i", "--input", "-o", "--output"]: normalized_args.append(token) if idx + 1 >= len(command_tokens): break mapped_path = _map_container_path_to_project( command_tokens[idx + 1], source_prefix, container_wdir, ) normalized_args.append(mapped_path) if token in ["-i", "--input"]: detected_inputs.append(mapped_path) else: detected_outputs.append(mapped_path) idx += 2 continue if token.startswith("--input=") or token.startswith("--output="): option, path_value = token.split("=", 1) mapped_path = _map_container_path_to_project( path_value, source_prefix, container_wdir, ) normalized_args.append(f"{option}={mapped_path}") if option == "--input": detected_inputs.append(mapped_path) else: detected_outputs.append(mapped_path) idx += 1 continue normalized_args.append(token) idx += 1 stage_name = None if is_mermaid_image and detected_inputs: raw_stage_name = f"mermaid-{Path(detected_inputs[0]).stem}" stage_name = _sanitize_stage_name(raw_stage_name) env_name = environment if env_name is None: if is_mermaid_image: env_name = "mermaid" else: env_name = image_name.split("/")[-1].replace("_", "-") description = "Mermaid CLI via Docker." if not is_mermaid_image: description = f"Docker CLI via image {image}." return NormalizedXRDockerCommand( image=image, wdir=container_wdir, command=normalized_args, inputs=detected_inputs, outputs=detected_outputs, environment_name=env_name, stage_name=stage_name, description=description, command_mode="entrypoint", ) def extract_docker_run_inner_command( cmd: str | list[str], ) -> list[str] | None: """Extract the inner command from a ``docker run ...`` invocation.""" if isinstance(cmd, str): try: tokens = shlex.split(cmd) except ValueError: return None else: tokens = cmd tokens = split_xr_command(tokens) parsed = _parse_docker_run_command(tokens) if parsed is None: return None inner_command: list[str] = parsed.get("command", []) if not inner_command: return None return inner_command def infer_xr_docker_environment( cmd: list[str], environment: str | None = None, ) -> tuple[str, dict] | None: """Infer a Docker environment from a `docker run ...` command. Returns a tuple of `(env_name, env_dict)` when parsing succeeds, or `None` for non-Docker/non-parseable commands. """ cmd = split_xr_command(cmd) parsed = _parse_docker_run_command(cmd) if parsed is None: return None image = _normalize_docker_image(parsed["image"]) image_name = _image_name_without_tag_or_digest(image) env_name = environment or image_name.split("/")[-1].replace("_", "-") wdir = parsed["workdir"] or "/work" env: dict = { "kind": "docker", "image": image, } if wdir != "/work": env["wdir"] = wdir if _uses_entrypoint_command_mode(image): env["command_mode"] = "entrypoint" return env_name, env # Keys from ``docker inspect`` that identify an image's content, as opposed # to metadata like creation time, which changes on every build LOCK_INSPECT_KEYS = ["RepoDigests", "Architecture", "Os", "RootFS"] # The registry used when an image reference has no registry component DEFAULT_REGISTRY = "docker.io" def inspect_image(ref: str) -> dict | None: """Return ``docker inspect`` output for an image, or None if absent.""" try: out = subprocess.check_output( ["docker", "inspect", ref], stderr=subprocess.DEVNULL ).decode() except (subprocess.CalledProcessError, FileNotFoundError): return None try: resp: list = json.loads(out) except json.JSONDecodeError: return None if not resp: return None first = resp[0] return first if isinstance(first, dict) else None def inspect_image_for_lock(ref: str) -> dict | None: """Return the identity-defining subset of an image's inspect output.""" resp = inspect_image(ref) if resp is None: return None return {key: resp.get(key) for key in LOCK_INSPECT_KEYS} def split_image_ref( ref: str, ) -> tuple[str | None, str, str | None, str | None]: """Split an image reference into registry, name, tag, and digest. The registry is None for references like ``ubuntu:22.04``, which Docker implicitly resolves against Docker Hub. """ digest = None if "@" in ref: ref, digest = ref.split("@", 1) registry = None remainder = ref if "/" in ref: first, rest = ref.split("/", 1) # A first component is only a registry if it looks like a host, i.e., # it has a dot or port separator, or is localhost. Otherwise it's a # Docker Hub namespace like 'library' in 'library/ubuntu'. if "." in first or ":" in first or first == "localhost": registry = first remainder = rest tag = None name = remainder last_slash = remainder.rfind("/") last_colon = remainder.rfind(":") if last_colon > last_slash: name = remainder[:last_colon] tag = remainder[last_colon + 1 :] return registry, name, tag, digest def get_default_registry_prefix(wdir: str | None = None) -> str | None: """Return the default registry prefix for a project's images. Images are namespaced under the project's GitHub repo in the GitHub Container Registry, e.g., ``ghcr.io/someone/some-project``. Returns None if the project has no GitHub remote, since there's then no namespace we can claim on the user's behalf. """ import calkit try: url = calkit.git.get_repo(wdir).remote().url except Exception: return None if "github.com" not in url: return None path = url.split("github.com")[-1].lstrip(":/").removesuffix(".git") parts = [p for p in path.split("/") if p] if len(parts) != 2: return None # Registry paths must be lowercase, whereas GitHub owner and repo names # are case-insensitive but case-preserving return "ghcr.io/" + "/".join(parts).lower() def get_remote_image_ref( image: str, registry_prefix: str, env_name: str | None = None ) -> str: """Build the remote reference an image is pushed to and pulled from. The local image name is kept as the final path component so a project's images stay distinguishable within its namespace, falling back to the environment name for images whose local name is already qualified. """ _, name, tag, _ = split_image_ref(image) name = name.rsplit("/", 1)[-1] if not name and env_name is not None: name = env_name # Repository names must be lowercase, but tags are case-sensitive, so # only the path is normalized path = f"{registry_prefix.rstrip('/')}/{name}".lower() return f"{path}:{tag or 'latest'}" def _run_showing_output(cmd: list[str]) -> tuple[bool, str]: """Run a command, showing its output as it happens and keeping it. Pushing and pulling an image are the slowest things Calkit does, and swallowing Docker's progress for minutes on end looks like a hang, so the output goes to the terminal as it arrives. It's captured too, since what a registry says on refusal decides what happens next. """ lines: list[str] = [] try: proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, ) except FileNotFoundError: return False, "Docker is not installed" assert proc.stdout is not None # Docker redraws each layer's status in place on a terminal, but writes # every repeat as its own line through a pipe, so a slow push scrolls # hundreds of identical 'Waiting' lines past. Only changes are worth # showing; a real transfer changes its byte count and still comes # through. last_status: dict[str, str] = {} for line in proc.stdout: lines.append(line) layer_id, sep, status = line.partition(": ") if sep and " " not in layer_id: if last_status.get(layer_id) == status: continue last_status[layer_id] = status print(line, end="", flush=True) proc.stdout.close() return proc.wait() == 0, "".join(lines) def pull_image(ref: str, platform: str | None = None) -> tuple[bool, str]: """Pull an image, returning success and its output.""" cmd = ["docker", "pull"] if platform is not None: cmd += ["--platform", platform] cmd.append(ref) return _run_showing_output(cmd) def pull_image_with_login(ref: str, platform: str | None = None) -> bool: """Pull an image, sorting out registry credentials if it's refused. Logging in replaces whatever credentials the machine already holds for that registry, which in CI are the ones the workflow logged in with, so it's only done when the registry actually refused this pull. An image that isn't there, or a registry that can't be reached, is not something a different set of credentials would fix. """ success, output = pull_image(ref, platform=platform) if success or not is_auth_error(output): return success logged_in, _ = login_to_registry(ref) if not logged_in: return False success, _ = pull_image(ref, platform=platform) return success def tag_image(source: str, target: str) -> bool: """Apply an additional tag to an image, returning True on success.""" try: subprocess.check_output( ["docker", "tag", source, target], stderr=subprocess.STDOUT ) except (subprocess.CalledProcessError, FileNotFoundError): return False return True def push_image(ref: str) -> tuple[bool, str]: """Push an image, returning success and its output.""" return _run_showing_output(["docker", "push", ref]) def platform_to_arch_name(platform: dict | str) -> str | None: """Convert an OCI platform into Calkit's lock file architecture name. Returns None for non-Linux platforms and for the ``unknown/unknown`` entries registries use for attestation manifests, neither of which describe a runnable image. """ if isinstance(platform, str): parts = platform.split("/") os_name = parts[0] if parts else "" arch = parts[1] if len(parts) > 1 else "" variant = parts[2] if len(parts) > 2 else "" else: os_name = platform.get("os", "") arch = platform.get("architecture", "") variant = platform.get("variant", "") or "" if os_name != "linux" or not arch or arch == "unknown": return None if variant: return f"{arch}-{variant}" return arch def inspect_remote_image(ref: str) -> dict | None: """Inspect an image in a registry without pulling it. Returns the parsed ``docker buildx imagetools inspect`` output, which carries the manifest (or index) and the image config for every platform the reference resolves to. """ try: out = subprocess.check_output( [ "docker", "buildx", "imagetools", "inspect", ref, "--format", "{{json .}}", ], stderr=subprocess.DEVNULL, ).decode() except (subprocess.CalledProcessError, FileNotFoundError): return None try: resp = json.loads(out) except json.JSONDecodeError: return None return resp if isinstance(resp, dict) else None def get_remote_image_platform_locks(ref: str) -> dict[str, dict] | None: """Read the identity of every platform behind a remote image reference. Returns a mapping of Calkit architecture name to the same identifying fields ``docker inspect`` provides, so lock files can be written for platforms this machine can't run. None means the registry couldn't be asked---it's unreachable, the image isn't there, or this machine has no buildx plugin---as opposed to an empty mapping, which is the registry answering that it serves no platform we can lock. Only an answer can make a lock file stale. """ resp = inspect_remote_image(ref) if resp is None: return None manifest = resp.get("manifest") or {} digest = manifest.get("digest") repo_digests = [digest] if digest else [] images = resp.get("image") or {} # A single-platform reference yields one config object rather than a # mapping of platform to config if "rootfs" in images: images = { f"{images.get('os')}/{images.get('architecture')}": images, } locks = {} for platform, config in images.items(): if not isinstance(config, dict): continue arch_name = platform_to_arch_name(platform) if arch_name is None: continue rootfs = config.get("rootfs") or {} diff_ids = rootfs.get("diff_ids") if not diff_ids: continue locks[arch_name] = { "RepoDigests": repo_digests, "Architecture": config.get("architecture"), "Os": config.get("os"), "RootFS": {"Type": "layers", "Layers": diff_ids}, } return locks def lock_matches_spec( lock: dict, dockerfile_md5: str | None, deps_md5s: dict[str, str], ) -> bool: """Return True if a lock file describes the current environment spec. A lock that doesn't match is stale rather than merely out-of-date with the local image: its recorded digest identifies an image built from different inputs, so it can't be used to pull. What the image is called isn't part of this. A lock belongs to the environment whose directory it sits in, the name comes from calkit.yaml, and the digest names the content, so renaming an image would only rerun every stage in it for software that didn't change. A digest left over from a different image can't resolve either, since the repo it's asked for comes from the current definition. """ if lock.get("DockerfileMD5") != dockerfile_md5: return False if (lock.get("DepsMD5s") or {}) != (deps_md5s or {}): return False return True def lock_matches_image(lock: dict, image_info: dict) -> bool: """Return True if an image's content is what a lock file records.""" locked = (lock.get("RootFS") or {}).get("Layers") actual = (image_info.get("RootFS") or {}).get("Layers") return bool(locked) and locked == actual def get_lock_digest_refs( lock: dict, remote_ref: str | None = None ) -> list[str]: """Return references that pull the image a lock records, best first. A lock records the digest alone, since a digest names the image's content while a repository only says where a copy of it lives. The repo to pull from comes from ``remote_ref``, worked out from the environment definition at the time of asking, so an environment that changes which image it uses can't resolve a digest left over from the old one. Locks written before digests were stored bare name their repo outright and are still honored. """ refs = [] remote_repo = ( get_repo_from_ref(remote_ref) if remote_ref is not None else None ) for digest in lock.get("RepoDigests") or []: if "@" in digest: refs.append(digest) elif remote_repo is not None and ":" in digest: refs.append(f"{remote_repo}@{digest}") if remote_repo is None: return refs preferred = [r for r in refs if r.split("@", 1)[0] == remote_repo] return preferred + [r for r in refs if r not in preferred] def build_lock( identity: dict, dockerfile_md5: str | None, deps_md5s: dict[str, str], run_config: dict, ) -> dict: """Assemble a lock file's contents for one platform. Key order is fixed so that a lock written for a platform from a registry matches byte-for-byte the one that platform would write for itself. """ lock = {key: identity.get(key) for key in LOCK_INSPECT_KEYS} # Normalize here rather than at each call site, so that a lock carried # over from an earlier run, in whatever form that run wrote it, comes # out in the same form as one written from scratch lock["RepoDigests"] = get_content_digests(lock) lock["DockerfileMD5"] = dockerfile_md5 lock["DepsMD5s"] = deps_md5s lock.update(run_config) return lock # Values of an environment's ``registry`` that name the GitHub Container # Registry without saying which namespace, leaving that to be worked out # from the project's Git remote. ``ghcr.io`` is the documented one, since a # registry Calkit runs itself someday would have as much claim to 'auto' AUTO_REGISTRY_VALUES = ["ghcr.io", "ghcr", "github", "auto", "true"] def registry_is_auto(registry: str | None) -> bool: """Return True if a ``registry`` value asks Calkit to work one out.""" if registry is None: return False return str(registry).strip().lower() in AUTO_REGISTRY_VALUES def resolve_registry_prefix(env: dict, wdir: str | None = None) -> str | None: """Return the registry prefix used for an environment's images. Registries are opt-in, since pushing an image publishes it somewhere the project doesn't necessarily control, so images are kept local when ``registry`` is unset or null. ``ghcr.io`` on its own resolves to the GitHub Container Registry namespace beside the project's repo, which is the one namespace we can name on the user's behalf. The strings a shell has to use in place of null are read as null too, since ``--registry none`` is the only way to say it on the command line. """ registry = env.get("registry") if registry is None or registry is False: return None registry = str(registry).strip() if registry.lower() in ["none", "false", ""]: return None if registry.lower() in AUTO_REGISTRY_VALUES: return get_default_registry_prefix(wdir=wdir) return registry def get_repo_from_ref(ref: str) -> str: """Return an image reference's repository, without its tag or digest.""" repo = ref.split("@", 1)[0] if ":" in repo.rsplit("/", 1)[-1]: repo = repo[: repo.rfind(":")] return repo def untag_image(ref: str) -> bool: """Remove one tag from an image, leaving the image itself alone.""" try: subprocess.check_output( ["docker", "rmi", "--no-prune", ref], stderr=subprocess.STDOUT ) except (subprocess.CalledProcessError, FileNotFoundError): return False return True def registry_has_image(remote_ref: str, identity: dict) -> bool: """Return True if a registry already serves this exact image. Tagging an image with a registry-qualified name is enough to give it a digest under that repo locally, so an image's own digests say nothing about whether anything was ever pushed. Only what the registry serves does, so its layers are what get compared. """ layers = (identity.get("RootFS") or {}).get("Layers") if not layers: return False remote = get_remote_image_platform_locks(remote_ref) if remote is None: return False return any( (entry.get("RootFS") or {}).get("Layers") == layers for entry in remote.values() ) def get_content_digests(identity: dict) -> list[str]: """Return the digests an image carries, without their repositories. A manifest is content-addressed, so an image built here already carries the digest it will have in a registry: pushing transfers those bytes rather than recomputing them. That makes a local build's digest worth recording before it has been pushed anywhere. """ digests = [] for entry in identity.get("RepoDigests") or []: _, _, digest = entry.rpartition("@") if ":" in digest and digest not in digests: digests.append(digest) return digests def keep_only_repo_digests(identity: dict, ref: str | None) -> dict: """Reduce an image's digests to the bare one a lock should record. Which digests an image carries locally depends on how it was obtained: building assigns one under a repo name that doesn't exist anywhere, and pulling by digest then tagging leaves both that and the real one. Only the digest the given reference's repo serves is kept, and only the ``sha256:...`` itself, since that names the image's content while the repository only says where a copy of it lives. Moving an environment to a different registry then leaves the lock alone, rather than rerunning every stage in it for software that didn't change. """ identity = dict(identity) if ref is None: identity["RepoDigests"] = [] return identity repo = get_repo_from_ref(ref) digests = [] for entry in identity.get("RepoDigests") or []: entry_repo, _, digest = entry.rpartition("@") if entry_repo and entry_repo != repo: continue # A digest is ``<algorithm>:<hex>``, so anything without a separator # is not one, whatever else it might be if ":" in digest and digest not in digests: digests.append(digest) identity["RepoDigests"] = digests return identity def get_lock_archs(env: dict) -> list[str]: """Return the architectures an environment should be locked for. Both of the architectures in common use are locked whether or not this machine runs them, so that moving a project between them doesn't invalidate every stage in the environment. """ from calkit.environments import DEFAULT_DOCKER_LOCK_ARCHS archs = list(DEFAULT_DOCKER_LOCK_ARCHS) for platform in env.get("build_platforms") or []: arch = platform_to_arch_name(platform) if arch is not None and arch not in archs: archs.append(arch) return archs def image_exists(ref: str) -> bool: """Return True if an image is present in the local image store.""" try: subprocess.check_output( ["docker", "image", "inspect", "--format", "{{.Id}}", ref], stderr=subprocess.DEVNULL, ) except (subprocess.CalledProcessError, FileNotFoundError): return False return True # What a registry says when the credentials it got aren't good enough, as # opposed to the network being down or the image not existing _AUTH_ERROR_MARKERS = [ "permission_denied", "does not match expected scopes", "insufficient_scope", "unauthorized", "authentication required", "denied:", "requested access to the resource is denied", ] # Creating a classic token with exactly what pushing an image needs GITHUB_PACKAGES_TOKEN_URL = ( "https://github.com/settings/tokens/new" "?scopes=write:packages,read:packages&description=Calkit" ) def is_auth_error(output: str) -> bool: """Return True if a registry refused the credentials it was given.""" lowered = output.lower() return any(marker in lowered for marker in _AUTH_ERROR_MARKERS) def get_github_token_scopes(token: str) -> set[str]: """Return the OAuth scopes a GitHub token carries. A GitHub App token and a fine-grained token both report none, so an empty set means "can't tell from here", not "can't do anything". """ import urllib.error import urllib.request req = urllib.request.Request( "https://api.github.com/user", headers={ "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json", }, ) try: with urllib.request.urlopen(req, timeout=15) as resp: raw = resp.headers.get("X-OAuth-Scopes") or "" except (urllib.error.URLError, OSError): return set() return {scope.strip() for scope in raw.split(",") if scope.strip()} def get_github_username() -> str: """Return the user's GitHub login, or a placeholder if unavailable. The GitHub Container Registry authenticates on the token, so the username only has to be non-empty. """ import calkit.github try: login = calkit.github.get("/user")["login"] except Exception: return "calkit" return str(login) if login else "calkit" def prompt_for_packages_token() -> str | None: """Walk the user through creating a token that can push packages. Only reached once the tokens Calkit already holds have been refused. The one it uses for the GitHub API can usually push, since its GitHub App can be granted permission to write packages, so a token the user creates is what's left when that permission isn't there. """ import webbrowser import typer typer.echo( "\nPushing to the GitHub Container Registry needs permission to " "write packages, which the token Calkit uses for the GitHub API " "was refused.\n\nOpening GitHub to create one with the " "'write:packages' scope already selected:\n" f" {GITHUB_PACKAGES_TOKEN_URL}\n" ) try: webbrowser.open(GITHUB_PACKAGES_TOKEN_URL) except Exception: pass entered: str = typer.prompt( "Paste the token here (input hidden)", hide_input=True, default="" ) token = entered.strip() if not token: return None scopes = get_github_token_scopes(token) if scopes and "write:packages" not in scopes: typer.echo( "That token doesn't have the 'write:packages' scope; it has: " + (", ".join(sorted(scopes)) or "none") ) return None return token def save_packages_token(token: str) -> None: """Remember a token that can push packages, so this is a one-off.""" from calkit import config cfg = config.read() cfg = config.Settings.model_validate( cfg.model_dump() | {"github_packages_token": token} ) cfg.write() def login_to_registry( ref: str, interactive: bool = False ) -> tuple[bool, str | None]: """Log in to a registry, reporting success and any new token to save. Only the GitHub Container Registry is handled, since that's the one Calkit has a path to credentials for. Anything else relies on the user's own ``docker login``. ``interactive`` asks the user for a token instead of trying the ones Calkit already holds. It's only reached once those have been tried and the push was still refused, so retrying them would just fail again. """ host = ref.split("/", 1)[0] if host != "ghcr.io": return False, None import calkit from_prompt = False token: str | None = None if interactive: token = prompt_for_packages_token() from_prompt = token is not None else: stored = calkit.config.read().github_packages_token token = str(stored) if stored is not None else None if token is None and not interactive: # The token Calkit already holds is worth trying: its GitHub App can # be granted permission to write packages, as can the token GitHub # Actions provides, and neither reports any OAuth scope, so whether # one can push is only settled by pushing with it try: token = calkit.github.get_token() except Exception: token = None if not token: return False, None try: subprocess.run( [ "docker", "login", host, "-u", get_github_username(), "--password-stdin", ], input=str(token).encode(), check=True, capture_output=True, ) except (subprocess.CalledProcessError, FileNotFoundError): return False, None return True, str(token) if from_prompt else None def push_image_with_login( ref: str, interactive: bool = False ) -> tuple[bool, str]: """Push an image, sorting out registry credentials if it's refused. A token is only remembered once a push has actually gone through with it, since logging in succeeds with credentials that still can't push. """ success, output = push_image(ref) if success or not is_auth_error(output): return success, output logged_in, _ = login_to_registry(ref) if logged_in: success, output = push_image(ref) if success or not is_auth_error(output): return success, output if not interactive: return success, output logged_in, new_token = login_to_registry(ref, interactive=True) if not logged_in: return success, output success, output = push_image(ref) if success and new_token is not None: save_packages_token(new_token) return success, output def get_image_name( env: dict, env_name: str, wdir: str | None = None ) -> str | None: """Return the name of the image an environment uses. An environment that builds from a Dockerfile doesn't have to be told what to call its image: the project and environment it belongs to name it unambiguously, by the same convention its Jupyter kernel is named by. One defined purely by an image has to name it, since it's someone else's, so there's nothing to work out and None comes back. None also comes back for a project with nothing to be named after: no ``owner`` or ``name`` in ``calkit.yaml`` and no Git remote to read them from. The directory a project happens to sit in is not a name---moving or renaming it would rename the image, and every image built by every project called ``analysis`` would collide---so this is for the project to say rather than for Calkit to guess. """ import calkit image: str | None = env.get("image") if image: return image if not env.get("path"): return None project: str | None try: project = calkit.detect_project_name(wdir=wdir) except ValueError: # A project that isn't published anywhere yet still names itself if # calkit.yaml says what it's called project = calkit.load_calkit_info(wdir=wdir).get("name") if not project: return None # Repository names must be lowercase, unlike tags return f"{project}.{env_name}".lower() def get_pushable_images(wdir: str | None = None) -> dict[str, dict]: """Return the images a project builds that belong in a registry. Only environments built from the project's own Dockerfile are included: an environment named after an image someone else publishes already lives somewhere it can be pulled back from. """ import calkit ck_info = calkit.load_calkit_info(wdir=wdir) resp = {} for env_name, env in (ck_info.get("environments") or {}).items(): if not isinstance(env, dict) or env.get("kind") != "docker": continue if not env.get("path"): continue image = get_image_name(env, env_name, wdir=wdir) if not image: continue prefix = resolve_registry_prefix(env, wdir=wdir) if prefix is None: continue resp[env_name] = dict( env=env, image=image, remote_ref=get_remote_image_ref(image, prefix), ) return resp
"""Functionality for working with Docker."""
from __future__ import annotations
import json
import os
import re
import shlex
import subprocess
from pathlib import Path
from pydantic import BaseModel
MINIFORGE_LAYER_TXT = r"""
# Install Miniforge
ARG MINIFORGE_NAME=Miniforge3
ARG MINIFORGE_VERSION=24.9.2-0
ARG TARGETPLATFORM
ENV CONDA_DIR=/opt/conda
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ENV PATH=${CONDA_DIR}/bin:${PATH}
# 1. Install just enough for conda to work
# 2. Keep $HOME clean (no .wget-hsts file), since HSTS isn't useful in this context
# 3. Install miniforge from GitHub releases
# 4. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html
# Particularly, we remove pyc and a files. The default install has no js, we can skip that
# 5. Activate base by default when running as any *non-root* user as well
# Good security practice requires running most workloads as non-root
# This makes sure any non-root users created also have base activated
# for their interactive shells.
# 6. Activate base by default when running as root as well
# The root user is already created, so won't pick up changes to /etc/skel
RUN apt-get update > /dev/null && \
apt-get install --no-install-recommends --yes \
wget bzip2 ca-certificates \
git \
tini \
> /dev/null && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \
/bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \
rm /tmp/miniforge.sh && \
conda clean --tarballs --index-cache --packages --yes && \
find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \
find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \
conda clean --force-pkgs-dirs --all --yes && \
echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
""".strip()
FOAMPY_LAYER_TEXT = r"""
RUN pip install --no-cache-dir numpy pandas matplotlib h5py \
&& pip install --no-cache-dir scipy \
&& pip install --no-cache-dir foampy
""".strip()
UV_LAYER_TEXT = """
COPY --from=ghcr.io/astral-sh/uv:0.8.5 /uv /uvx /bin/
"""
JULIA_LAYER_TEXT = """
# Install Julia
# Ensure base image is a bullseye distribution
COPY --from=julia:1.11.6-bullseye /usr/local/julia /usr/local/julia
ENV JULIA_PATH=/usr/local/julia \
PATH=$PATH:/usr/local/julia/bin \
JULIA_GPG=3673DF529D9049477F76B37566E3C7DC03D6E495 \
JULIA_VERSION=1.11.6
"""
LAYERS = {
"mambaforge": MINIFORGE_LAYER_TXT,
"miniforge": MINIFORGE_LAYER_TXT,
"foampy": FOAMPY_LAYER_TEXT,
"uv": UV_LAYER_TEXT,
"julia": JULIA_LAYER_TEXT,
}
# Docker images whose commands should be passed directly to the image
# entrypoint when normalizing `xr` commands
XR_DOCKER_ENTRYPOINT_IMAGES = {
"minlag/mermaid-cli",
}
class NormalizedXRDockerCommand(BaseModel):
"""Normalized Docker command metadata for `calkit xr`."""
image: str
wdir: str
command: list[str]
inputs: list[str]
outputs: list[str]
environment_name: str
stage_name: str | None = None
description: str | None = None
command_mode: str = "shell"

Xet Storage Details

Size:
44.2 kB
·
Xet hash:
3a823f35201218ec90b5df9970e558bb5427dcad92a91bd154a11648ea9b839c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.