Buckets:
| """Functionality for handling artifact provenance.""" | |
| from __future__ import annotations | |
| import functools | |
| import os | |
| import re | |
| # The artifact kinds whose provenance is *checked*. Each entry must say | |
| # where it came from: a pipeline stage, an import, or the person who | |
| # created it, e.g., by collecting or measuring the data or drawing the | |
| # figure. | |
| # | |
| # Deliberately not the same set as the kinds an import can be *recorded* | |
| # in, which is read off the models by ``get_artifact_types_with_imports``. | |
| # A notebook can carry an ``imported_from`` and be refreshed like anything | |
| # else, but notebooks are usually written by a project's own authors, so | |
| # holding every one of them to a provenance record would report gaps that | |
| # aren't. | |
| PROVENANCE_ARTIFACT_TYPES = [ | |
| "datasets", | |
| "figures", | |
| "publications", | |
| "tables", | |
| "presentations", | |
| "misc", | |
| ] | |
| def get_artifact_types_with_imports() -> list[str]: | |
| """The ``calkit.yaml`` lists whose entries can record an import. | |
| Read off the models, so a kind that gains ``imported_from`` is | |
| searched without anything here changing. Wider than | |
| :data:`PROVENANCE_ARTIFACT_TYPES`, which is about which kinds are held | |
| to having provenance at all --- an entry that says where it came from | |
| is honored wherever it is written, whether or not its kind is one we | |
| would report for staying silent. | |
| """ | |
| from typing import get_args | |
| from pydantic import BaseModel | |
| from calkit.models.core import ProjectInfo | |
| types = [] | |
| for kind, field in ProjectInfo.model_fields.items(): | |
| for arg in get_args(field.annotation): | |
| for model in (arg, *get_args(arg)): | |
| if ( | |
| isinstance(model, type) | |
| and issubclass(model, BaseModel) | |
| and "imported_from" in model.model_fields | |
| ): | |
| types.append(kind) | |
| break | |
| else: | |
| continue | |
| break | |
| return types | |
| def get_importable_artifact_types() -> list[str]: | |
| """The kinds whose model takes ``imported_from``. | |
| Read off the models rather than listed by hand: whether a kind can | |
| record an import is a fact about its model, and a copy of that fact | |
| would keep validating long after it stopped being true. Adding the | |
| field to tables or presentations is then all it takes to make them | |
| importable. | |
| Computed on demand rather than at import: reading the models pulls in | |
| pydantic, and this module is imported on every CLI invocation. | |
| """ | |
| from typing import get_args | |
| from pydantic import BaseModel | |
| from calkit.models.core import ProjectInfo | |
| types = [] | |
| for kind in PROVENANCE_ARTIFACT_TYPES: | |
| field = ProjectInfo.model_fields.get(kind) | |
| if field is None: | |
| continue | |
| # Each list is annotated as list[<Model>], possibly through a | |
| # union of the imported and non-imported forms | |
| for arg in get_args(field.annotation): | |
| for model in (arg, *get_args(arg)): | |
| if ( | |
| isinstance(model, type) | |
| and issubclass(model, BaseModel) | |
| and "imported_from" in model.model_fields | |
| ): | |
| types.append(kind) | |
| break | |
| else: | |
| continue | |
| break | |
| return types | |
| # Where the resolved state of each import is recorded: the commit a Git | |
| # source actually landed on, a checksum of what was fetched, and when. | |
| # Committed, so everyone cloning the project gets the same bytes, the way | |
| # they do from ``dvc.lock`` or an environment's lock file --- the same | |
| # reason ``.calkit/overleaf-sync.json`` is committed, since two people | |
| # syncing the same Overleaf project have to agree on where they last | |
| # were. What this file records is one-way: an import has no other side to | |
| # reconcile with, so there is only ever what was fetched. | |
| IMPORT_LOCK_FPATH = os.path.join(".calkit", "imports.json") | |
| def read_import_locks(wdir: str | None = None) -> dict: | |
| """Read the recorded state of every import, keyed by path.""" | |
| import json | |
| fpath = ( | |
| os.path.join(wdir, IMPORT_LOCK_FPATH) if wdir else (IMPORT_LOCK_FPATH) | |
| ) | |
| if not os.path.isfile(fpath): | |
| return {} | |
| with open(fpath) as f: | |
| try: | |
| locks = json.load(f) | |
| except ValueError as e: | |
| # Reported rather than treated as absent: the next write would | |
| # replace the file, so swallowing this turns a file somebody | |
| # can still fix into every import losing its recorded state | |
| raise ValueError( | |
| f"{fpath} is not valid JSON ({e}); fix or delete it -- " | |
| "treating it as empty would discard every import's " | |
| "recorded state on the next write" | |
| ) | |
| # Early versions wrote a list of Zenodo import events here. Nothing | |
| # ever read it, so it is treated as absent rather than migrated. | |
| if not isinstance(locks, dict): | |
| return {} | |
| return locks | |
| def write_import_lock( | |
| path: str, lock: dict | None, wdir: str | None = None | |
| ) -> str: | |
| """Record what an import resolved to, or drop it with ``lock=None``. | |
| Returns the lock file's path, so the caller can commit it alongside | |
| whatever it fetched. | |
| """ | |
| import json | |
| fpath = ( | |
| os.path.join(wdir, IMPORT_LOCK_FPATH) if wdir else (IMPORT_LOCK_FPATH) | |
| ) | |
| locks = read_import_locks(wdir=wdir) | |
| if lock is None: | |
| locks.pop(path, None) | |
| else: | |
| # 'fetched' says when this version arrived, not when it was last | |
| # checked for. Refreshing an unchanged import must leave the file | |
| # alone, or every check would be a commit and nothing would ever | |
| # read as up to date. | |
| previous = locks.get(path) | |
| if previous is not None and { | |
| k: v for k, v in previous.items() if k != "fetched" | |
| } == {k: v for k, v in lock.items() if k != "fetched"}: | |
| lock = dict(lock) | |
| if "fetched" in previous: | |
| lock["fetched"] = previous["fetched"] | |
| locks[path] = lock | |
| os.makedirs(os.path.dirname(fpath), exist_ok=True) | |
| # Sorted so the file is stable across runs and diffs read as changes to | |
| # one import rather than a reshuffle | |
| # newline="\n" so the file is byte-identical on every platform; it is | |
| # committed, and CRLF on Windows would churn the diff for everyone else | |
| with open(fpath, "w", newline="\n") as f: | |
| json.dump(locks, f, indent=2, sort_keys=True) | |
| f.write("\n") | |
| return fpath | |
| # Named in the recorded value rather than in the key it is stored under, | |
| # so it can change without the lock file's shape changing --- and so a | |
| # hash written by another version is recognizable as one this version | |
| # can't compare against. | |
| HASH_ALGORITHM = "sha256" | |
| def hash_path(path: str) -> str | None: | |
| """Checksum what is at ``path``, for telling whether it has changed. | |
| A directory is hashed over its entries as well as their contents, so a | |
| renamed or removed file changes the digest. Symlinks are hashed by | |
| their target rather than followed, since following one would reach | |
| outside the tree. Returns None only when there is nothing there. | |
| A Git source can bring in a directory, and refreshing one replaces it | |
| wholesale, so it needs a checksum for the same reason a file does: | |
| without one, an edit inside it would be deleted without warning. | |
| """ | |
| import hashlib | |
| digest = hashlib.sha256() | |
| def add_file(fpath: str) -> None: | |
| with open(fpath, "rb") as f: | |
| for chunk in iter(lambda: f.read(65536), b""): | |
| digest.update(chunk) | |
| if os.path.islink(path): | |
| digest.update(b"link\0" + os.readlink(path).encode()) | |
| elif os.path.isfile(path): | |
| add_file(path) | |
| elif os.path.isdir(path): | |
| # Sorted so the digest doesn't depend on directory order, and | |
| # names are hashed too so a rename is a change | |
| for root, dirs, files in os.walk(path): | |
| dirs.sort() | |
| for name in sorted(files + dirs): | |
| full = os.path.join(root, name) | |
| rel = os.path.relpath(full, path).replace(os.sep, "/") | |
| digest.update(rel.encode() + b"\0") | |
| if os.path.islink(full): | |
| digest.update(b"link\0" + os.readlink(full).encode()) | |
| elif os.path.isfile(full): | |
| add_file(full) | |
| else: | |
| return None | |
| return f"{HASH_ALGORITHM}:{digest.hexdigest()}" | |
| def local_edit(path: str, lock: dict | None) -> bool: | |
| """Whether what is on disk differs from what the import last fetched. | |
| False when there is nothing to compare against -- an entry written | |
| before hashes were recorded, or one recorded with an algorithm this | |
| version doesn't compute -- since a refresh that can't tell shouldn't | |
| claim it can. | |
| """ | |
| recorded = (lock or {}).get("hash") | |
| if not recorded or not os.path.exists(path): | |
| return False | |
| # The algorithm is named in the value, so a file hashed by a version | |
| # that used a different one is not mistaken for an edited file | |
| algorithm = recorded.split(":", 1)[0] | |
| if algorithm != HASH_ALGORITHM: | |
| return False | |
| return hash_path(path) != recorded | |
| def check_project_path(path: str) -> str: | |
| """Return why ``path`` isn't safe to write in the project, or "". | |
| An import writes to this path and later hands it to ``git add``, so a | |
| hand-written ``../..`` or an absolute path would put a file outside | |
| the repo and then fail confusingly. Symlinks are resolved, since one | |
| inside the project can still point out of it. | |
| """ | |
| if os.path.isabs(path): | |
| return f"'{path}' must be a path inside the project, not absolute" | |
| root = os.path.realpath(os.getcwd()) | |
| full = os.path.realpath(os.path.join(root, path)) | |
| if full != root and os.path.commonpath([root, full]) != root: | |
| return f"'{path}' points outside the project" | |
| return "" | |
| def artifact_kind_to_list(kind: str) -> str: | |
| """The ``calkit.yaml`` list a singular artifact kind is recorded in. | |
| Kinds are named in the singular where a person types one -- a figure, | |
| a dataset -- and the lists holding them are plural. ``misc`` is | |
| already a mass noun, so it is spelled the same either way. | |
| """ | |
| return kind if kind == "misc" else kind + "s" | |
| def get_importable_artifact_kinds() -> list[str]: | |
| """Singular names of the kinds an import can be recorded as. | |
| The same set as :func:`get_importable_artifact_types`, spelled the way | |
| the CLI takes them, so the two can't name different things. | |
| """ | |
| return [ | |
| kind if kind == "misc" else kind.removesuffix("s") | |
| for kind in get_importable_artifact_types() | |
| ] | |
| def has_provenance(artifact: dict) -> bool: | |
| """Return whether an artifact entry records where it came from. | |
| A stage and an import are the stronger forms, but ``created_by`` counts | |
| too: a dataset someone measured, or a schematic someone drew, is | |
| accounted for even though there's nothing upstream to point at. The | |
| field names in ``calkit.reproducibility.ReproCheck`` predate | |
| attribution and are kept so callers reading them keep working. | |
| """ | |
| return any( | |
| artifact.get(key) is not None | |
| for key in ["stage", "imported_from", "created_by"] | |
| ) | |
| def find_artifact(ck_info: dict, path: str) -> tuple[str, dict] | None: | |
| """Find the entry recorded at ``path``, whichever list it's in. | |
| Returns the list's name and the entry, so the caller can edit it in | |
| place. | |
| """ | |
| for kind in get_artifact_types_with_imports(): | |
| for entry in ck_info.get(kind, []) or []: | |
| if isinstance(entry, dict) and entry.get("path") == path: | |
| return kind, entry | |
| return None | |
| # Hosts whose file URLs name a Git repo, a revision, and a path within it, | |
| # so a link copied out of the browser can be imported as the Git source it | |
| # actually is rather than as an HTML page that happens to contain the file. | |
| _GIT_FORGE_HOSTS = ("github.com", "gitlab.com", "raw.githubusercontent.com") | |
| # The segment that separates the ref from the path in a forge URL. GitLab | |
| # additionally puts '/-/' before it, since its group paths nest. | |
| _FORGE_REF_MARKERS = ("blob", "raw", "tree", "blame", "src") | |
| def _git_source_from_url(url: str, ref: str | None = None) -> dict | None: | |
| """Read a Git repo, revision, and path out of a forge URL. | |
| Returns None for a URL that isn't one, which is then just a URL to | |
| download. Guessing is confined to hosts whose layout is known: for | |
| anything else, the bytes at the address are what was asked for. | |
| A ref can contain slashes, and the URL doesn't say where it ends and | |
| the path begins. It's taken to be one segment unless ``ref`` says | |
| what it is, in which case the path is whatever follows it. | |
| """ | |
| from urllib.parse import unquote, urlsplit | |
| parts = urlsplit(url) | |
| host = parts.netloc.lower().removeprefix("www.") | |
| if host not in _GIT_FORGE_HOSTS: | |
| return None | |
| # Decoded, since a browser writes a space as '%20' and the checkout | |
| # doesn't | |
| segments = [unquote(s) for s in parts.path.split("/") if s] | |
| # GitLab separates the repo from what you're looking at inside it with | |
| # '/-/', because its groups nest and there is otherwise no telling | |
| # where the repo path ends | |
| if "-" in segments: | |
| split_at = segments.index("-") | |
| repo_segments, rest = segments[:split_at], segments[split_at + 1 :] | |
| elif host == "raw.githubusercontent.com": | |
| # Always <owner>/<repo>/<ref>/<path>, with no marker segment | |
| if len(segments) < 3: | |
| return None | |
| repo_segments, rest = segments[:2], ["raw"] + segments[2:] | |
| else: | |
| if len(segments) < 2: | |
| return None | |
| repo_segments, rest = segments[:2], segments[2:] | |
| if not rest: | |
| return None | |
| url_ref: str | None = None | |
| if rest[0] in _FORGE_REF_MARKERS and len(rest) >= 3: | |
| url_ref, path_segments = rest[1], rest[2:] | |
| # GitHub's raw URLs spell a branch as 'refs/heads/<name>' | |
| if url_ref == "refs" and len(path_segments) >= 2: | |
| if path_segments[0] in ("heads", "tags"): | |
| url_ref, path_segments = path_segments[1], path_segments[2:] | |
| ref_segments = ref.split("/") if ref else [] | |
| if len(ref_segments) > 1 and rest[1 : 1 + len(ref_segments)] == ( | |
| ref_segments | |
| ): | |
| path_segments = rest[1 + len(ref_segments) :] | |
| else: | |
| # No revision named, e.g. a path written out by hand. The default | |
| # branch is what gets fetched, and the commit it resolves to is | |
| # what gets recorded. | |
| path_segments = rest | |
| if ref is None: | |
| ref = url_ref | |
| if not path_segments: | |
| return None | |
| scheme = parts.scheme or "https" | |
| forge_host = "github.com" if host == "raw.githubusercontent.com" else host | |
| repo = "/".join(repo_segments).removesuffix(".git") | |
| source: dict = { | |
| "git_repo_url": f"{scheme}://{forge_host}/{repo}.git", | |
| "path": "/".join(path_segments), | |
| } | |
| # Recorded as a 'git_ref' whether it names a branch, a tag, or a | |
| # commit. A commit is a thing to follow that happens never to move, | |
| # which is what keeps a deliberate pin pinned when refreshed. | |
| if ref is not None: | |
| source["git_ref"] = ref | |
| return source | |
| # Schemes git understands that aren't HTTP. A clone URL is the most | |
| # natural thing to paste, so it's recognized as the Git source it is | |
| # rather than falling through to the Calkit-project reading, where | |
| # 'git@github.com:o/r' would be sent to the hub as a project name. | |
| _GIT_URL_SCHEMES = ("ssh://", "git://", "git+ssh://") | |
| # The scp-like form, 'git@github.com:owner/repo/path'. The host is required | |
| # to contain a dot so an ordinary relative path with a colon in it isn't | |
| # mistaken for one. | |
| _SCP_URL_RE = re.compile( | |
| r"^(?P<user>[^@/\s]+@)?(?P<host>[A-Za-z0-9._-]+\.[A-Za-z0-9._-]+)" | |
| r":(?P<path>[^/].*)$" | |
| ) | |
| def _git_source_from_ssh_url(url: str, ref: str | None = None) -> dict | None: | |
| """Read a repo and a path within it out of an SSH-style clone URL. | |
| Handles both the scp-like ``git@host:owner/repo/path`` and the | |
| ``ssh://git@host/owner/repo/path`` form, and returns None for anything | |
| that is neither. | |
| Unlike a forge's web URL, a clone URL has no marker saying where the | |
| repo ends and a path inside it begins, so the repo is taken to be the | |
| first two segments. That is right for GitHub and for GitLab projects | |
| that aren't in nested groups; for anything else, naming the repo with | |
| ``--git-repo`` says it exactly rather than leaving it to be guessed. | |
| """ | |
| from urllib.parse import urlsplit | |
| scp_prefix = None | |
| if url.startswith(_GIT_URL_SCHEMES): | |
| parts = urlsplit(url) | |
| host = parts.netloc | |
| rest = parts.path.lstrip("/") | |
| scheme = parts.scheme | |
| else: | |
| match = _SCP_URL_RE.match(url) | |
| if match is None: | |
| return None | |
| host = (match.group("user") or "") + match.group("host") | |
| rest = match.group("path") | |
| scheme = None | |
| scp_prefix = host | |
| segments = [seg for seg in rest.split("/") if seg] | |
| if len(segments) < 2: | |
| return None | |
| repo_segments, path_segments = segments[:2], segments[2:] | |
| repo = "/".join(repo_segments).removesuffix(".git") | |
| # Written back in the form it was given, since that is what git was | |
| # going to be handed either way and what the person will recognize | |
| if scp_prefix is not None: | |
| repo_url = f"{scp_prefix}:{repo}.git" | |
| else: | |
| repo_url = f"{scheme}://{host}/{repo}.git" | |
| source: dict = {"git_repo_url": repo_url} | |
| if path_segments: | |
| source["path"] = "/".join(path_segments) | |
| if ref is not None: | |
| source["git_ref"] = ref | |
| return source | |
| def source_from_location( | |
| location: str, | |
| git_repo: str | None = None, | |
| ref: str | None = None, | |
| ) -> dict: | |
| """Work out where a file is being imported from, from how it's written. | |
| ``location`` is a path inside ``git_repo`` when that's given, and | |
| otherwise a URL, an SSH clone URL like | |
| ``git@github.com:owner/repo/path``, a DOI, or a Calkit project path | |
| like ``someone/some-project/scripts/setup.sh``. An explicit ``ref`` names | |
| the branch, tag, or commit to follow, and overrides one read out of a | |
| URL. Without one, the repo's default branch is what gets fetched, now | |
| and whenever the import is refreshed. | |
| """ | |
| if git_repo is not None: | |
| source: dict = {"git_repo_url": git_repo, "path": location} | |
| if ref is not None: | |
| source["git_ref"] = ref | |
| return source | |
| stripped = re.sub( | |
| r"^(https?://(dx\.)?doi\.org/|doi:)", "", location.strip(), flags=re.I | |
| ) | |
| if re.fullmatch(r"10\.\d{4,9}/\S+", stripped): | |
| # Recognized rather than downloaded: a DOI resolves to a landing | |
| # page, so treating it as a plain URL would save the HTML and call | |
| # it the data. 'fetch' says what to use instead. | |
| return {"doi": stripped} | |
| if location.startswith(("http://", "https://")): | |
| git_source = _git_source_from_url(location, ref=ref) | |
| if git_source is not None: | |
| return git_source | |
| # A link to a Zenodo record is that record, not a file. Read as the | |
| # DOI it stands for, since the two name the same thing and a record | |
| # is a landing page: downloading it would save the HTML and call it | |
| # the data, which is the mistake the DOI branch above exists to | |
| # prevent. Zenodo's version DOI is its record ID, which is how | |
| # 'calkit import zenodo' reads one in the other direction. | |
| record_match = re.match( | |
| r"^https?://(www\.)?zenodo\.org/records?/(?P<id>\d+)", | |
| location, | |
| flags=re.I, | |
| ) | |
| if record_match is not None: | |
| return {"doi": f"10.5281/zenodo.{record_match.group('id')}"} | |
| return {"url": location} | |
| ssh_source = _git_source_from_ssh_url(location, ref=ref) | |
| if ssh_source is not None: | |
| return ssh_source | |
| segments = location.split("/") | |
| if len(segments) < 3: | |
| raise ValueError( | |
| f"Cannot tell where '{location}' comes from; give a URL, a DOI, " | |
| "a Calkit project path like someone/some-project/path/to/file, " | |
| "or a path inside a repo named with --git-repo" | |
| ) | |
| return {"project": "/".join(segments[:2]), "path": "/".join(segments[2:])} | |
| def git_source(imported_from: dict) -> dict | None: | |
| """Read a Git source out of an entry, or None if it isn't one.""" | |
| if imported_from.get("git_repo_url") is None: | |
| return None | |
| return { | |
| k: imported_from.get(k) | |
| for k in ("git_repo_url", "path", "git_ref", "git_rev") | |
| } | |
| def default_dest_path(imported_from: dict) -> str: | |
| """Where an import lands when no destination was given.""" | |
| if (git := git_source(imported_from)) is not None: | |
| return os.path.basename(str(git.get("path") or "")) | |
| if imported_from.get("project") is not None: | |
| return str(imported_from.get("path") or "") | |
| if (url := imported_from.get("url")) is not None: | |
| from urllib.parse import urlsplit | |
| return os.path.basename(urlsplit(str(url)).path) | |
| return "" | |
| def describe_source(imported_from: dict) -> str: | |
| """Name where an artifact came from, for a message to a person.""" | |
| if (git := git_source(imported_from)) is not None: | |
| path = git.get("path") | |
| where = str(git.get("git_repo_url") or "a Git repo") | |
| return where + (f"/{path}" if path else "") | |
| if (project := imported_from.get("project")) is not None: | |
| path = imported_from.get("path") | |
| return str(project) + (f"/{path}" if path else "") | |
| if (url := imported_from.get("url")) is not None: | |
| return str(url) | |
| if (doi := imported_from.get("doi")) is not None: | |
| return f"doi:{doi}" | |
| return "an unrecorded source" | |
| def _rev_exists(repo_dir: str, rev: str) -> bool: | |
| """Return whether ``rev`` names something in the clone at ``repo_dir``. | |
| The clone has no checkout and no local branches, so a branch is only | |
| there as ``origin/<name>``. That's what ``git checkout`` finds by | |
| DWIM, and it has to be looked for explicitly here: ``rev-parse`` | |
| resolves ``refs/remotes/<name>``, which a branch called ``feature/foo`` | |
| is not. | |
| """ | |
| import subprocess | |
| for candidate in (rev, f"origin/{rev}"): | |
| if ( | |
| subprocess.call( | |
| [ | |
| "git", | |
| "-C", | |
| repo_dir, | |
| "rev-parse", | |
| "--verify", | |
| "--quiet", | |
| candidate, | |
| ], | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL, | |
| ) | |
| == 0 | |
| ): | |
| return True | |
| return False | |
| def _widen_ref(repo_dir: str, ref: str, path: str) -> tuple[str, str] | None: | |
| """Move the ref/path boundary rightwards until the ref resolves. | |
| A forge URL for a branch called ``feature/foo`` reads as | |
| ``.../blob/feature/foo/scripts/a.sh``, and nothing in it says whether | |
| ``foo`` is the rest of the branch name or the first directory. The | |
| boundary is guessed at one segment when the URL is parsed; here, with | |
| the repo in hand, the guess can be checked and corrected. | |
| Returns the corrected ``(ref, path)``, or None if no split resolves. | |
| The shortest widening wins, so a repo with both ``feature`` and | |
| ``feature/foo`` keeps the reading the URL most likely meant. | |
| """ | |
| segments = [s for s in path.split("/") if s] | |
| for n in range(1, len(segments)): | |
| candidate = "/".join([ref, *segments[:n]]) | |
| rest = "/".join(segments[n:]) | |
| if _ref_and_path_resolve(repo_dir, candidate, rest): | |
| return candidate, rest | |
| return None | |
| def _ref_and_path_resolve(repo_dir: str, ref: str, path: str) -> bool: | |
| """Whether ``ref`` exists *and* holds ``path``. | |
| Checked as a pair. A repo with both ``feature`` and ``feature/foo`` | |
| would otherwise stop at ``feature``, which resolves, and then look for | |
| a path that only exists on ``feature/foo`` -- the split has to be | |
| judged by whether the whole thing works, not by the ref alone. | |
| """ | |
| import subprocess | |
| if not _rev_exists(repo_dir, ref): | |
| return False | |
| if not path: | |
| return True | |
| for candidate in (ref, f"origin/{ref}"): | |
| if ( | |
| subprocess.call( | |
| [ | |
| "git", | |
| "-C", | |
| repo_dir, | |
| "cat-file", | |
| "-e", | |
| f"{candidate}:{path}", | |
| ], | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL, | |
| ) | |
| == 0 | |
| ): | |
| return True | |
| return False | |
| def _with_state(lock: dict, dest_path: str, rev: str | None = None) -> dict: | |
| """Fill in what a fetch resolved to: the commit, checksum, and when.""" | |
| from datetime import datetime, timezone | |
| if rev is not None: | |
| lock["rev"] = rev | |
| checksum = hash_path(dest_path) | |
| if checksum is not None: | |
| lock["hash"] = checksum | |
| lock["fetched"] = ( | |
| datetime.now(timezone.utc).replace(microsecond=0).isoformat() | |
| ) | |
| return lock | |
| def fetch(imported_from: dict, dest_path: str) -> tuple[dict, dict]: | |
| """Download what an ``imported_from`` entry points at, to ``dest_path``. | |
| Returns the entry as it should now be recorded, and separately what the | |
| fetch resolved to: the commit a Git source landed on, a checksum of the | |
| file, and when. The first belongs in ``calkit.yaml`` because a person | |
| wrote it; the second belongs in ``.calkit/imports.json`` because the | |
| tool worked it out. | |
| What gets fetched is the source's ``ref``---a branch, a tag, or a | |
| commit---or the repo's default branch when it names none. The recorded | |
| ``rev`` is the answer, never the question: refreshing an import is | |
| asking where the thing it follows is now, and reading the last answer | |
| back would make that a no-op. | |
| Whatever is at ``dest_path`` is replaced. This is a one-way copy from | |
| the source, not a merge: an import is a statement about where a file | |
| came from, and a local edit that survived it would make that false. | |
| """ | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| imported_from = dict(imported_from) | |
| lock: dict = {} | |
| os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True) | |
| if (git := git_source(imported_from)) is not None: | |
| repo_url = git.get("git_repo_url") | |
| if not repo_url: | |
| raise ValueError("Git source has no 'repo_url' to fetch from") | |
| src_path = git.get("path") | |
| if not src_path: | |
| raise ValueError( | |
| "Git source has no 'path'; importing a whole repo is what a " | |
| "Git submodule is for" | |
| ) | |
| # What to check out. Not 'rev': that records what a previous fetch | |
| # returned, so using it here would make every refresh return the | |
| # same thing it did last time. Nothing named means the repo's | |
| # default branch. | |
| target = git.get("git_ref") | |
| with tempfile.TemporaryDirectory() as tmp_dir: | |
| # Blobless and checkout-less, so a big repo costs about as | |
| # little as fetching one file from it can | |
| clone_cmd = [ | |
| "git", | |
| "clone", | |
| "--filter=blob:none", | |
| "--no-checkout", | |
| "--quiet", | |
| repo_url, | |
| tmp_dir, | |
| ] | |
| try: | |
| subprocess.check_call(clone_cmd) | |
| except subprocess.CalledProcessError as e: | |
| raise ValueError(f"Failed to clone {repo_url}: {e}") | |
| # A branch name can contain slashes, and a forge URL doesn't say | |
| # where the ref ends and the path begins, so the guess made when | |
| # the URL was read may have cut it in the wrong place. Only | |
| # reached when the ref as recorded doesn't resolve, so an | |
| # explicit --git-ref that works is never second-guessed. | |
| if target is not None and not _ref_and_path_resolve( | |
| tmp_dir, target, src_path | |
| ): | |
| widened = _widen_ref(tmp_dir, target, src_path) | |
| if widened is not None: | |
| target, src_path = widened | |
| git["git_ref"], git["path"] = target, src_path | |
| try: | |
| subprocess.check_call( | |
| [ | |
| "git", | |
| "-C", | |
| tmp_dir, | |
| "checkout", | |
| "--quiet", | |
| target or "HEAD", | |
| ] | |
| ) | |
| rev = subprocess.check_output( | |
| ["git", "-C", tmp_dir, "rev-parse", "HEAD"], text=True | |
| ).strip() | |
| except subprocess.CalledProcessError as e: | |
| raise ValueError( | |
| f"Failed to fetch {src_path} from {repo_url}: {e}" | |
| ) | |
| # Resolved, so neither a '..' in the path nor a symlink in the | |
| # checkout can reach outside the clone | |
| clone_root = os.path.realpath(tmp_dir) | |
| full_src = os.path.realpath(os.path.join(clone_root, src_path)) | |
| if os.path.commonpath([clone_root, full_src]) != clone_root: | |
| raise ValueError(f"'{src_path}' points outside of {repo_url}") | |
| if not os.path.exists(full_src): | |
| raise ValueError( | |
| f"'{src_path}' does not exist in {repo_url} at " | |
| f"{target or rev}" | |
| ) | |
| if os.path.isdir(full_src): | |
| shutil.rmtree(dest_path, ignore_errors=True) | |
| # Symlinks are copied as symlinks rather than followed: the | |
| # containment check above covers the path asked for, but a | |
| # link *inside* the tree could still point anywhere on this | |
| # machine, and copying its target would pull a local file | |
| # into the project as though it came from the repo | |
| shutil.copytree(full_src, dest_path, symlinks=True) | |
| else: | |
| shutil.copyfile(full_src, dest_path) | |
| # What was actually fetched, so the import is reproducible even | |
| # when it follows a moving branch. Kept out of the entry: the entry | |
| # says what to follow, this says where that led. | |
| git.pop("git_rev", None) | |
| imported_from.update({k: v for k, v in git.items() if v is not None}) | |
| return imported_from, _with_state(lock, dest_path, rev=rev) | |
| if (url := imported_from.get("url")) is not None: | |
| import requests | |
| # Written beside the destination and moved over it once complete, | |
| # so a download that dies partway leaves the old file intact. | |
| # Every failure is reported the same way, including a connection | |
| # that never opens or dies mid-body: a caller refreshing many | |
| # imports has to be able to skip this one and carry on. | |
| part_path = dest_path + ".part" | |
| try: | |
| with requests.get(url, stream=True) as resp: | |
| resp.raise_for_status() | |
| with open(part_path, "wb") as f: | |
| for chunk in resp.iter_content(chunk_size=8192): | |
| f.write(chunk) | |
| os.replace(part_path, dest_path) | |
| except requests.RequestException as e: | |
| raise ValueError(f"Failed to download {url}: {e}") | |
| except OSError as e: | |
| raise ValueError(f"Failed to write {dest_path}: {e}") | |
| finally: | |
| if os.path.exists(part_path): | |
| os.remove(part_path) | |
| return imported_from, _with_state(lock, dest_path) | |
| if (project := imported_from.get("project")) is not None: | |
| import base64 | |
| import calkit | |
| src_path = imported_from.get("path") | |
| if not src_path: | |
| raise ValueError("Project source has no 'path' to fetch") | |
| try: | |
| contents: dict = calkit.hub.get( | |
| f"/projects/{project}/contents/{src_path}" | |
| ) | |
| except Exception as e: | |
| raise ValueError(f"Failed to fetch {src_path} from {project}: {e}") | |
| content = contents.get("content") | |
| if content is not None: | |
| with open(dest_path, "wb") as f: | |
| f.write(base64.b64decode(content)) | |
| return imported_from, _with_state(lock, dest_path) | |
| download_url = contents.get("url") | |
| if download_url is None: | |
| raise ValueError(f"Could not fetch {src_path} from {project}") | |
| # Big files come back as a link to fetch rather than inline, so the | |
| # URL branch above finishes the job. The entry still records the | |
| # project, since that is where the file came from. | |
| fetch({"url": download_url}, dest_path=dest_path) | |
| return imported_from, _with_state(lock, dest_path) | |
| if imported_from.get("doi") is not None: | |
| raise ValueError( | |
| "Fetching by DOI is not supported here, since a DOI resolves to " | |
| "a record rather than to a file; use 'calkit import zenodo' for " | |
| "a Zenodo record" | |
| ) | |
| raise ValueError( | |
| "Nothing to fetch from: expected one of 'git', 'url', 'project', or " | |
| "'doi'" | |
| ) | |
Xet Storage Details
- Size:
- 33.9 kB
- Xet hash:
- d7bf94f18784c784bedfa371dd5be5d72d0da069e3e65f67b63b329602c9597b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.