Buckets:
| """Working with LaTeX documents.""" | |
| from __future__ import annotations | |
| import os | |
| import re | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING | |
| from calkit.core import LOCAL_DIR | |
| if TYPE_CHECKING: | |
| # Only ever named in annotations here, which this module's | |
| # ``from __future__ import annotations`` leaves unevaluated. Importing | |
| # GitPython for real runs 'git version' twice as a side effect of the | |
| # import, and this module is reached from the CLI's own import chain -- | |
| # so every 'calkit' invocation paid for two git subprocesses to satisfy | |
| # a type hint. | |
| import git | |
| # Where revisions are checked out and the marked-up document is built. | |
| # Inside the project so a containerized TeX environment, which only sees | |
| # the working directory, can read them; under .calkit/local, which is | |
| # private to the machine and gitignored wholesale, so none of it is | |
| # mistaken for the diffs themselves. | |
| DIFF_TMP_DIR = os.path.join(LOCAL_DIR, "latex-diff-build") | |
| # Hashes of the marked-up source each diff was last built from, so a run | |
| # that would produce the same document again can skip the build. Machine | |
| # private, so a fresh clone simply builds once. | |
| DIFF_STATE_DIR = os.path.join(DIFF_TMP_DIR, "state") | |
| DIFF_AUX_DIR = os.path.join(DIFF_TMP_DIR, "aux") | |
| DIFF_DIR = os.path.join(".calkit", "latex-diffs") | |
| # Revisions that mean something different tomorrow. A comparison with one | |
| # of these at either end can't be settled by looking at files alone. | |
| MOVING_REFS = frozenset({"HEAD"}) | |
| # Where a comparison against the working tree goes. It can't be | |
| # reproduced from two commits, so it isn't something to track: it's a | |
| # development aid with a lifetime of minutes, and .calkit/local is | |
| # private to the machine. | |
| LOCAL_DIFF_DIR = os.path.join(LOCAL_DIR, "latex-diffs") | |
| # What the working tree is called when a comparison is named after its | |
| # ends | |
| WORKING_NAME = "working" | |
| def _ref_dirname(ref: str) -> str: | |
| """Turn a ref into something that can be one path component.""" | |
| name = ref.lstrip("_").replace("_", "-").replace("/", "-") | |
| return re.sub(r"[^A-Za-z0-9.-]+", "-", name).strip("-") | |
| def get_diff_dir(from_ref: str, to_ref: str | None = None) -> str: | |
| """The directory holding a comparison, named for the pair as written. | |
| Named from the spec rather than what it resolved to, since a stage's | |
| outputs have to be the same paths on every branch. | |
| A comparison against the working tree lives under the machine-private | |
| directory instead: it can't be reproduced from two commits, so it | |
| isn't something to keep. | |
| """ | |
| name = _ref_dirname(from_ref) | |
| if to_ref is None: | |
| # Against the working tree | |
| return Path( | |
| os.path.join(LOCAL_DIFF_DIR, f"{name}..{WORKING_NAME}") | |
| ).as_posix() | |
| if to_ref != "HEAD": | |
| # HEAD is what a comparison runs up to unless it says otherwise, | |
| # so naming it would only add noise | |
| name += f"..{_ref_dirname(to_ref)}" | |
| return Path(os.path.join(DIFF_DIR, name)).as_posix() | |
| def get_diff_path( | |
| tex_file: str, | |
| from_ref: str, | |
| to_ref: str | None = None, | |
| as_posix: bool = True, | |
| output_dir: str | None = None, | |
| ) -> str: | |
| """Return where a document's diff between two revisions is kept. | |
| Beside the other things Calkit derives from a project's files rather | |
| than next to the document, following executed notebooks: it's an | |
| output, and a PDF, so saving the project tracks it with DVC and its | |
| history comes along with the project's. | |
| A directory per pair, named for the pair as written rather than what | |
| it resolved to, since a stage's outputs have to be the same paths on | |
| every branch. The document's own path lives inside it, so two | |
| documents both called main.tex don't collide, and so the inputs to a | |
| comparison have somewhere to sit later. | |
| """ | |
| if output_dir is None: | |
| output_dir = get_diff_dir(from_ref, to_ref) | |
| p = os.path.join( | |
| output_dir, os.path.dirname(tex_file), Path(tex_file).stem + ".pdf" | |
| ) | |
| return Path(p).as_posix() if as_posix else p | |
| def diff_stage_suffix(from_ref: str, to_ref: str | None = None) -> str: | |
| """Name the DVC stage that builds a diff, from the pair as written.""" | |
| suffix = _ref_dirname(from_ref) | |
| if to_ref is not None and to_ref != "HEAD": | |
| suffix += f"-{_ref_dirname(to_ref)}" | |
| return suffix | |
| def diff_state_path(output: str) -> str: | |
| """Where the hash of a diff's marked-up source is remembered.""" | |
| flat = Path(output).as_posix().replace("/", "-") | |
| return os.path.join(DIFF_STATE_DIR, f"{flat}.sha256") | |
| def default_base_ref(repo: git.Repo) -> str: | |
| """What a change is naturally read against: the merge base with the | |
| default branch. | |
| Not the default branch itself, since work that landed there after this | |
| branch started isn't part of this change and would otherwise show up | |
| as deletions. Used when nobody says what to compare against; a | |
| pipeline names the branch itself, which is more readable and doesn't | |
| depend on which machine resolved it. | |
| """ | |
| import warnings | |
| candidates = [] | |
| try: | |
| candidates.append(repo.remotes.origin.refs.HEAD.reference.name) | |
| except Exception: | |
| pass | |
| candidates += ["origin/main", "origin/master", "main", "master"] | |
| for candidate in candidates: | |
| try: | |
| base = str(repo.git.merge_base("HEAD", candidate)).strip() | |
| except Exception: | |
| continue | |
| if base: | |
| return base | |
| warnings.warn("Could not find a default branch; comparing against HEAD~1") | |
| return "HEAD~1" | |
| # Extensions tried, in order, when a reference omits one. LaTeX resolves | |
| # \includegraphics{fig} against the graphics extension list, and | |
| # \input{sec}/\bibliography{refs} against .tex/.bib. | |
| _GRAPHICS_EXTS = [".pdf", ".png", ".jpg", ".jpeg", ".eps", ".ps", ".gif"] | |
| # Commands that name a local file, mapped to the extensions to try. Each | |
| # takes a comma-separated list in its final braces group. | |
| _INPUT_COMMANDS: dict[str, list[str]] = { | |
| "documentclass": [".cls"], | |
| "usepackage": [".sty"], | |
| "RequirePackage": [".sty"], | |
| "bibliographystyle": [".bst"], | |
| "bibliography": [".bib"], | |
| "addbibresource": [".bib"], | |
| "input": [".tex"], | |
| "include": [".tex"], | |
| "subfile": [".tex"], | |
| "includegraphics": _GRAPHICS_EXTS, | |
| "includesvg": [".svg", ".pdf"], | |
| "lstinputlisting": [""], | |
| "verbatiminput": [""], | |
| } | |
| # Files that are themselves LaTeX source, so their own inputs count too. | |
| _SOURCE_EXTS = frozenset({".tex", ".cls", ".sty", ".clo", ".def", ".cfg"}) | |
| # One command with optional [...] args, then its {...} argument. TeX | |
| # comments are stripped first, so a % here is a literal one. | |
| _INPUT_RE = re.compile( | |
| r"\\(" + "|".join(_INPUT_COMMANDS) + r")\s*(?:\[[^\]]*\])*\s*\{([^}]*)\}" | |
| ) | |
| def _strip_comments(tex: str) -> str: | |
| """Remove TeX comments, keeping escaped percent signs.""" | |
| out = [] | |
| for line in tex.splitlines(): | |
| out.append(re.sub(r"(?<!\\)%.*$", "", line)) | |
| return "\n".join(out) | |
| def _project_has(base: Path, rel: str) -> bool: | |
| """Whether the project contains this file. | |
| A DVC-tracked figure isn't in the working tree until it's pulled, but | |
| its ``.dvc`` pointer is always in Git, and it's every bit as much a | |
| project file -- and a stage input -- as one stored directly. Without | |
| this, detection would depend on whether the caller had pulled, and a | |
| server working from a fresh clone would miss every DVC-tracked figure. | |
| """ | |
| return (base / rel).is_file() or (base / f"{rel}.dvc").is_file() | |
| def detect_inputs(target_path: str, wdir: str | None = None) -> list[str]: | |
| """Find the project files a LaTeX document reads. | |
| A document's class, style, bibliography, and figure files are inputs | |
| to building it, but LaTeX names them without paths or extensions and | |
| resolves them itself, so nothing in the pipeline sees them unless | |
| they're declared. Undeclared, a change to the class file doesn't | |
| rebuild the paper, and the in-browser preview -- which only has the | |
| files the stage declares -- can't compile at all. | |
| Only files that exist in the project are returned; everything else | |
| (``graphicx``, ``natbib``, and the rest of TeX Live) comes from the | |
| TeX installation and isn't ours to track. Source files found this way | |
| are read in turn, so a document split across files contributes its | |
| whole tree, and a journal class that loads its own style files (the | |
| JFM template's ``jfm.cls`` pulls in ``upmath.sty`` and | |
| ``lineno-FLM.sty``) contributes those too. | |
| Paths are returned relative to ``wdir`` -- the same frame as | |
| ``target_path`` and the rest of a stage's paths -- sorted, and with | |
| the target itself excluded (it's already a dependency). | |
| """ | |
| base = Path(wdir) if wdir else Path(".") | |
| root = Path(target_path) | |
| found: set[str] = set() | |
| # Source files whose own inputs still need collecting, and everything | |
| # already visited, so a pair of files including each other terminates. | |
| queue = [root] | |
| seen = {root.as_posix()} | |
| while queue: | |
| current = queue.pop() | |
| try: | |
| tex = _strip_comments( | |
| (base / current).read_text(encoding="utf-8", errors="replace") | |
| ) | |
| except (OSError, UnicodeDecodeError): | |
| continue | |
| for command, arg in _INPUT_RE.findall(tex): | |
| for raw in [n.strip() for n in arg.split(",")]: | |
| if not raw or raw.startswith(("http://", "https://")): | |
| continue | |
| # TeX resolves a reference against the compile's working | |
| # directory, which is the root document's; a nested file | |
| # written as if paths were relative to itself is common | |
| # enough to try too. Names as written come first, since a | |
| # reference can already carry its extension. | |
| names = [raw] + [ | |
| raw + ext for ext in _INPUT_COMMANDS[command] if ext | |
| ] | |
| candidates = [ | |
| parent / name | |
| for name in names | |
| for parent in dict.fromkeys([root.parent, current.parent]) | |
| ] | |
| for candidate in candidates: | |
| rel = Path(os.path.normpath(candidate)).as_posix() | |
| if rel.startswith("..") or not _project_has(base, rel): | |
| continue | |
| if rel != root.as_posix(): | |
| found.add(rel) | |
| if Path(rel).suffix in _SOURCE_EXTS and rel not in seen: | |
| seen.add(rel) | |
| queue.append(Path(rel)) | |
| break | |
| return sorted(found) | |
| def _is_immutable_ref(repo: git.Repo, ref: str | None) -> bool: | |
| """Whether a ref names something that can't change under us. | |
| A tag or a commit hash pins content; a branch or the working tree | |
| doesn't. Only a diff between two of the former can be built once and | |
| left alone. | |
| """ | |
| if ref is None: | |
| return False | |
| if ref in [tag.name for tag in repo.tags]: | |
| return True | |
| if ref in [head.name for head in repo.heads]: | |
| return False | |
| if not re.fullmatch(r"[0-9a-f]{7,40}", ref): | |
| return False | |
| try: | |
| repo.commit(ref) | |
| except Exception: | |
| return False | |
| return True | |
Xet Storage Details
- Size:
- 11.4 kB
- Xet hash:
- 08dcf22f3cedf2d061c3f65812c0a59d28c43fc748fb7d6f4c5ada9ee52709e6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.