Buckets:

download
raw
11.4 kB
"""Working with LaTeX documents."""
from __future__ import annotations
import os
import re
from pathlib import Path
from typing import TYPE_CHECKING
from calkit.core import LOCAL_DIR
if TYPE_CHECKING:
# Only ever named in annotations here, which this module's
# ``from __future__ import annotations`` leaves unevaluated. Importing
# GitPython for real runs 'git version' twice as a side effect of the
# import, and this module is reached from the CLI's own import chain --
# so every 'calkit' invocation paid for two git subprocesses to satisfy
# a type hint.
import git
# Where revisions are checked out and the marked-up document is built.
# Inside the project so a containerized TeX environment, which only sees
# the working directory, can read them; under .calkit/local, which is
# private to the machine and gitignored wholesale, so none of it is
# mistaken for the diffs themselves.
DIFF_TMP_DIR = os.path.join(LOCAL_DIR, "latex-diff-build")
# Hashes of the marked-up source each diff was last built from, so a run
# that would produce the same document again can skip the build. Machine
# private, so a fresh clone simply builds once.
DIFF_STATE_DIR = os.path.join(DIFF_TMP_DIR, "state")
DIFF_AUX_DIR = os.path.join(DIFF_TMP_DIR, "aux")
DIFF_DIR = os.path.join(".calkit", "latex-diffs")
# Revisions that mean something different tomorrow. A comparison with one
# of these at either end can't be settled by looking at files alone.
MOVING_REFS = frozenset({"HEAD"})
# Where a comparison against the working tree goes. It can't be
# reproduced from two commits, so it isn't something to track: it's a
# development aid with a lifetime of minutes, and .calkit/local is
# private to the machine.
LOCAL_DIFF_DIR = os.path.join(LOCAL_DIR, "latex-diffs")
# What the working tree is called when a comparison is named after its
# ends
WORKING_NAME = "working"
def _ref_dirname(ref: str) -> str:
"""Turn a ref into something that can be one path component."""
name = ref.lstrip("_").replace("_", "-").replace("/", "-")
return re.sub(r"[^A-Za-z0-9.-]+", "-", name).strip("-")
def get_diff_dir(from_ref: str, to_ref: str | None = None) -> str:
"""The directory holding a comparison, named for the pair as written.
Named from the spec rather than what it resolved to, since a stage's
outputs have to be the same paths on every branch.
A comparison against the working tree lives under the machine-private
directory instead: it can't be reproduced from two commits, so it
isn't something to keep.
"""
name = _ref_dirname(from_ref)
if to_ref is None:
# Against the working tree
return Path(
os.path.join(LOCAL_DIFF_DIR, f"{name}..{WORKING_NAME}")
).as_posix()
if to_ref != "HEAD":
# HEAD is what a comparison runs up to unless it says otherwise,
# so naming it would only add noise
name += f"..{_ref_dirname(to_ref)}"
return Path(os.path.join(DIFF_DIR, name)).as_posix()
def get_diff_path(
tex_file: str,
from_ref: str,
to_ref: str | None = None,
as_posix: bool = True,
output_dir: str | None = None,
) -> str:
"""Return where a document's diff between two revisions is kept.
Beside the other things Calkit derives from a project's files rather
than next to the document, following executed notebooks: it's an
output, and a PDF, so saving the project tracks it with DVC and its
history comes along with the project's.
A directory per pair, named for the pair as written rather than what
it resolved to, since a stage's outputs have to be the same paths on
every branch. The document's own path lives inside it, so two
documents both called main.tex don't collide, and so the inputs to a
comparison have somewhere to sit later.
"""
if output_dir is None:
output_dir = get_diff_dir(from_ref, to_ref)
p = os.path.join(
output_dir, os.path.dirname(tex_file), Path(tex_file).stem + ".pdf"
)
return Path(p).as_posix() if as_posix else p
def diff_stage_suffix(from_ref: str, to_ref: str | None = None) -> str:
"""Name the DVC stage that builds a diff, from the pair as written."""
suffix = _ref_dirname(from_ref)
if to_ref is not None and to_ref != "HEAD":
suffix += f"-{_ref_dirname(to_ref)}"
return suffix
def diff_state_path(output: str) -> str:
"""Where the hash of a diff's marked-up source is remembered."""
flat = Path(output).as_posix().replace("/", "-")
return os.path.join(DIFF_STATE_DIR, f"{flat}.sha256")
def default_base_ref(repo: git.Repo) -> str:
"""What a change is naturally read against: the merge base with the
default branch.
Not the default branch itself, since work that landed there after this
branch started isn't part of this change and would otherwise show up
as deletions. Used when nobody says what to compare against; a
pipeline names the branch itself, which is more readable and doesn't
depend on which machine resolved it.
"""
import warnings
candidates = []
try:
candidates.append(repo.remotes.origin.refs.HEAD.reference.name)
except Exception:
pass
candidates += ["origin/main", "origin/master", "main", "master"]
for candidate in candidates:
try:
base = str(repo.git.merge_base("HEAD", candidate)).strip()
except Exception:
continue
if base:
return base
warnings.warn("Could not find a default branch; comparing against HEAD~1")
return "HEAD~1"
# Extensions tried, in order, when a reference omits one. LaTeX resolves
# \includegraphics{fig} against the graphics extension list, and
# \input{sec}/\bibliography{refs} against .tex/.bib.
_GRAPHICS_EXTS = [".pdf", ".png", ".jpg", ".jpeg", ".eps", ".ps", ".gif"]
# Commands that name a local file, mapped to the extensions to try. Each
# takes a comma-separated list in its final braces group.
_INPUT_COMMANDS: dict[str, list[str]] = {
"documentclass": [".cls"],
"usepackage": [".sty"],
"RequirePackage": [".sty"],
"bibliographystyle": [".bst"],
"bibliography": [".bib"],
"addbibresource": [".bib"],
"input": [".tex"],
"include": [".tex"],
"subfile": [".tex"],
"includegraphics": _GRAPHICS_EXTS,
"includesvg": [".svg", ".pdf"],
"lstinputlisting": [""],
"verbatiminput": [""],
}
# Files that are themselves LaTeX source, so their own inputs count too.
_SOURCE_EXTS = frozenset({".tex", ".cls", ".sty", ".clo", ".def", ".cfg"})
# One command with optional [...] args, then its {...} argument. TeX
# comments are stripped first, so a % here is a literal one.
_INPUT_RE = re.compile(
r"\\(" + "|".join(_INPUT_COMMANDS) + r")\s*(?:\[[^\]]*\])*\s*\{([^}]*)\}"
)
def _strip_comments(tex: str) -> str:
"""Remove TeX comments, keeping escaped percent signs."""
out = []
for line in tex.splitlines():
out.append(re.sub(r"(?<!\\)%.*$", "", line))
return "\n".join(out)
def _project_has(base: Path, rel: str) -> bool:
"""Whether the project contains this file.
A DVC-tracked figure isn't in the working tree until it's pulled, but
its ``.dvc`` pointer is always in Git, and it's every bit as much a
project file -- and a stage input -- as one stored directly. Without
this, detection would depend on whether the caller had pulled, and a
server working from a fresh clone would miss every DVC-tracked figure.
"""
return (base / rel).is_file() or (base / f"{rel}.dvc").is_file()
def detect_inputs(target_path: str, wdir: str | None = None) -> list[str]:
"""Find the project files a LaTeX document reads.
A document's class, style, bibliography, and figure files are inputs
to building it, but LaTeX names them without paths or extensions and
resolves them itself, so nothing in the pipeline sees them unless
they're declared. Undeclared, a change to the class file doesn't
rebuild the paper, and the in-browser preview -- which only has the
files the stage declares -- can't compile at all.
Only files that exist in the project are returned; everything else
(``graphicx``, ``natbib``, and the rest of TeX Live) comes from the
TeX installation and isn't ours to track. Source files found this way
are read in turn, so a document split across files contributes its
whole tree, and a journal class that loads its own style files (the
JFM template's ``jfm.cls`` pulls in ``upmath.sty`` and
``lineno-FLM.sty``) contributes those too.
Paths are returned relative to ``wdir`` -- the same frame as
``target_path`` and the rest of a stage's paths -- sorted, and with
the target itself excluded (it's already a dependency).
"""
base = Path(wdir) if wdir else Path(".")
root = Path(target_path)
found: set[str] = set()
# Source files whose own inputs still need collecting, and everything
# already visited, so a pair of files including each other terminates.
queue = [root]
seen = {root.as_posix()}
while queue:
current = queue.pop()
try:
tex = _strip_comments(
(base / current).read_text(encoding="utf-8", errors="replace")
)
except (OSError, UnicodeDecodeError):
continue
for command, arg in _INPUT_RE.findall(tex):
for raw in [n.strip() for n in arg.split(",")]:
if not raw or raw.startswith(("http://", "https://")):
continue
# TeX resolves a reference against the compile's working
# directory, which is the root document's; a nested file
# written as if paths were relative to itself is common
# enough to try too. Names as written come first, since a
# reference can already carry its extension.
names = [raw] + [
raw + ext for ext in _INPUT_COMMANDS[command] if ext
]
candidates = [
parent / name
for name in names
for parent in dict.fromkeys([root.parent, current.parent])
]
for candidate in candidates:
rel = Path(os.path.normpath(candidate)).as_posix()
if rel.startswith("..") or not _project_has(base, rel):
continue
if rel != root.as_posix():
found.add(rel)
if Path(rel).suffix in _SOURCE_EXTS and rel not in seen:
seen.add(rel)
queue.append(Path(rel))
break
return sorted(found)
def _is_immutable_ref(repo: git.Repo, ref: str | None) -> bool:
"""Whether a ref names something that can't change under us.
A tag or a commit hash pins content; a branch or the working tree
doesn't. Only a diff between two of the former can be built once and
left alone.
"""
if ref is None:
return False
if ref in [tag.name for tag in repo.tags]:
return True
if ref in [head.name for head in repo.heads]:
return False
if not re.fullmatch(r"[0-9a-f]{7,40}", ref):
return False
try:
repo.commit(ref)
except Exception:
return False
return True

Xet Storage Details

Size:
11.4 kB
·
Xet hash:
08dcf22f3cedf2d061c3f65812c0a59d28c43fc748fb7d6f4c5ada9ee52709e6

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.