Buckets:
"""Functionality for working with runnable Markdown files.
A Markdown file can declare pipeline stages and environments by annotating
its code blocks, so a README can be the source of truth for what it
documents. An annotated block looks like::
```python calkit stage name=example environment=main
print("hello")
```
The first token of a fence's info string stays the language, since
Markdown renderers use it for syntax highlighting and ignore the rest.
Annotations can also be written as an HTML comment directive immediately
preceding a block, which keeps long attribute lists out of the info string
and lets a plain bulleted list declare an environment's dependencies::
<!-- calkit environment name=main python=3.13 -->
- numpy
- matplotlib
"""
from __future__ import annotations
import os
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
import ruamel.yaml
from pydantic import BaseModel
DirectiveKind = Literal["stage", "environment", "output", "value", "values"]
# ``output`` blocks are written to, never read: a stage's extracted script
# must not include what that stage printed, or injecting output would make
# the stage stale, which would rerun it, which would inject again.
# ``value`` and ``values`` are inline markers rather than block
# annotations; see ``set_values``.
DIRECTIVE_KINDS: tuple[str, ...] = (
"stage",
"environment",
"output",
"value",
"values",
)
BlockSource = Literal["fence", "list"]
# A fence is at least three backticks or tildes, optionally indented. Only
# the run of fence characters and the info string that follows are captured;
# the run's length matters because a longer fence can contain shorter ones,
# which is how a Markdown file documents this very feature without
# declaring the examples it shows.
_FENCE_RE = re.compile(
r"^(?P<indent>[ ]{0,3})(?P<fence>`{3,}|~{3,})(?P<info>.*)$"
)
_LIST_ITEM_RE = re.compile(r"^[ ]{0,3}[-*+][ ]+(?P<item>.*?)\s*$")
_COMMENT_OPEN_RE = re.compile(r"^[ ]{0,3}<!--\s*(?P<body>.*)$")
_yaml = ruamel.yaml.YAML(typ="safe")
class MarkdownParseError(ValueError):
"""Raised when a Calkit annotation in a Markdown file can't be read."""
def __init__(self, message: str, path: str | None = None, line: int = 0):
self.path = path
self.line = line
where = f"{path}:{line}" if path else f"line {line}"
super().__init__(f"{where}: {message}")
# Names are used as path components---a stage's extracted script and an
# environment's spec directory are named for them---so they must be a
# single safe segment rather than something that escapes the project.
_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
def check_name(name: str, what: str, path: str | None, line: int) -> None:
"""Check that a declared name can be used as a single path segment."""
if not _NAME_RE.match(name) or name in (".", ".."):
raise MarkdownParseError(
f"{what} name '{name}' is not valid; names may only contain "
"letters, digits, '.', '_' and '-', and must start with a "
"letter or digit",
path=path,
line=line,
)
@dataclass
class MarkdownBlock:
"""A Calkit-annotated block extracted from a Markdown file."""
kind: DirectiveKind
attrs: dict[str, Any] = field(default_factory=dict)
content: str = ""
language: str | None = None
source: BlockSource = "fence"
# 1-indexed line at which the block's annotation starts, for error
# messages that point back at what the author wrote.
line: int = 0
@property
def name(self) -> str | None:
name = self.attrs.get("name")
return None if name is None else str(name)
def _split_value(text: str, i: int) -> tuple[str, int]:
"""Read one attribute value out of ``text`` starting at index ``i``.
Values may be bracketed or braced (and so contain whitespace, as in
``outputs=[{path: fig.png, storage: git}]``), quoted, or a bare token
running to the next whitespace.
"""
openers = {"[": "]", "{": "}", "(": ")"}
if text[i] in openers:
depth = 0
quote: str | None = None
start = i
while i < len(text):
c = text[i]
if quote is not None:
if c == quote:
quote = None
elif c in "\"'":
quote = c
elif c in openers:
depth += 1
elif c in openers.values():
depth -= 1
if depth == 0:
return text[start : i + 1], i + 1
i += 1
raise ValueError(f"Unbalanced '{text[start]}' in attribute value")
if text[i] in "\"'":
quote = text[i]
start = i
i += 1
while i < len(text):
if text[i] == quote:
return text[start : i + 1], i + 1
i += 1
raise ValueError("Unterminated quote in attribute value")
start = i
while i < len(text) and not text[i].isspace():
i += 1
return text[start:i], i
def parse_attrs(text: str) -> dict[str, Any]:
"""Parse ``key=value`` attributes from an annotation.
Values are read as YAML flow scalars, so ``outputs=[{path: fig.png,
storage: git}]`` becomes real data without a bespoke grammar. A bare
key with no ``=`` is a flag and parses as ``True``.
"""
attrs: dict[str, Any] = {}
i = 0
while i < len(text):
if text[i].isspace():
i += 1
continue
key_start = i
while i < len(text) and not text[i].isspace() and text[i] != "=":
i += 1
key = text[key_start:i]
if not key:
raise ValueError(f"Empty attribute name near '{text[i:][:20]}'")
if i < len(text) and text[i] == "=":
i += 1
if i >= len(text) or text[i].isspace():
raise ValueError(f"Attribute '{key}' has no value")
raw, i = _split_value(text, i)
try:
value = _yaml.load(raw)
except Exception as e:
raise ValueError(
f"Failed to parse value for attribute '{key}': {raw}"
) from e
else:
value = True
if key in attrs:
raise ValueError(f"Duplicate attribute '{key}'")
attrs[key] = value
return attrs
def parse_annotation(text: str) -> tuple[DirectiveKind, dict[str, Any]] | None:
"""Parse a ``calkit <kind> ...`` annotation, if ``text`` is one.
Returns ``None`` when the text isn't a Calkit annotation at all, which
is the common case: an ordinary fence info string like ``python`` or a
normal HTML comment.
"""
tokens = text.strip().split(None, 1)
if not tokens:
return None
if tokens[0] != "calkit":
return None
rest = tokens[1] if len(tokens) > 1 else ""
parts = rest.split(None, 1)
if not parts or parts[0] not in DIRECTIVE_KINDS:
raise ValueError(
"Calkit annotation must name a directive "
f"({', '.join(DIRECTIVE_KINDS)}), got: {rest or '(nothing)'}"
)
kind: DirectiveKind
if parts[0] == "stage":
kind = "stage"
elif parts[0] == "environment":
kind = "environment"
elif parts[0] == "output":
kind = "output"
elif parts[0] == "value":
kind = "value"
else:
kind = "values"
return kind, parse_attrs(parts[1] if len(parts) > 1 else "")
def _parse_fence_info(
info: str,
) -> tuple[str | None, tuple[DirectiveKind, dict[str, Any]] | None]:
"""Split a fence info string into its language and Calkit annotation.
Renderers take the first token as the language and ignore the rest, so
``python calkit stage name=x`` still highlights as Python on GitHub.
"""
info = info.strip()
if not info:
return None, None
tokens = info.split(None, 1)
if tokens[0] == "calkit":
return None, parse_annotation(info)
language = tokens[0]
rest = tokens[1] if len(tokens) > 1 else ""
return language, parse_annotation(rest)
def _read_comment_directive(
lines: list[str], i: int
) -> tuple[tuple[DirectiveKind, dict[str, Any]] | None, int]:
"""Read an HTML comment directive starting at line ``i``.
Returns the parsed annotation (or ``None`` if this comment isn't one)
and the index of the first line after the comment. Comments may span
multiple lines so a long attribute list doesn't have to.
"""
m = _COMMENT_OPEN_RE.match(lines[i])
if m is None:
return None, i
body_parts = []
j = i
body = m.group("body")
while True:
end = body.find("-->")
if end != -1:
body_parts.append(body[:end])
j += 1
break
body_parts.append(body)
j += 1
if j >= len(lines):
# An unterminated comment isn't ours to complain about; the file
# is just Markdown with a typo, and every renderer swallows it.
return None, i + 1
body = lines[j]
return parse_annotation(" ".join(body_parts)), j
def _read_list_items(lines: list[str], i: int) -> tuple[list[str], int]:
"""Read a run of bullet list items starting at line ``i``."""
items = []
while i < len(lines):
m = _LIST_ITEM_RE.match(lines[i])
if m is None:
break
items.append(m.group("item"))
i += 1
return items, i
def parse_markdown(text: str, path: str | None = None) -> list[MarkdownBlock]:
"""Extract Calkit-annotated blocks from Markdown text.
Unannotated fences are inert, which is what makes it safe to keep
shell snippets and deliberately-wrong examples in the same file.
"""
lines = text.splitlines()
blocks: list[MarkdownBlock] = []
pending: tuple[DirectiveKind, dict[str, Any]] | None = None
pending_line = 0
i = 0
while i < len(lines):
line = lines[i]
fence_match = _FENCE_RE.match(line)
if fence_match is not None:
fence = fence_match.group("fence")
indent = len(fence_match.group("indent"))
try:
language, annotation = _parse_fence_info(
fence_match.group("info")
)
except ValueError as e:
raise MarkdownParseError(str(e), path=path, line=i + 1)
# Collect the body, closing only on a fence of the same
# character that is at least as long as the opening one. A
# shorter run inside is content, which is how this feature gets
# documented in Markdown without the examples going live.
body: list[str] = []
j = i + 1
while j < len(lines):
close = _FENCE_RE.match(lines[j])
if (
close is not None
and close.group("fence")[0] == fence[0]
and len(close.group("fence")) >= len(fence)
and not close.group("info").strip()
):
break
body.append(lines[j][indent:] if indent else lines[j])
j += 1
if annotation is not None and annotation[0] in ("value", "values"):
raise MarkdownParseError(
f"'calkit {annotation[0]}' is an inline marker, not a "
"code block annotation",
path=path,
line=i + 1,
)
if annotation is not None or pending is not None:
try:
kind, attrs = _merge(pending, annotation)
except ValueError as e:
raise MarkdownParseError(str(e), path=path, line=i + 1)
blocks.append(
MarkdownBlock(
kind=kind,
attrs=attrs,
content="\n".join(body),
language=language,
source="fence",
line=pending_line if pending is not None else i + 1,
)
)
pending = None
i = j + 1
continue
if pending is not None:
if not line.strip():
i += 1
continue
items, end = _read_list_items(lines, i)
if items:
blocks.append(
MarkdownBlock(
kind=pending[0],
attrs=pending[1],
content="\n".join(items),
language=None,
source="list",
line=pending_line,
)
)
pending = None
i = end
continue
raise MarkdownParseError(
"A Calkit directive comment must be followed by a fenced "
"code block or a bulleted list",
path=path,
line=pending_line,
)
try:
annotation, end = _read_comment_directive(lines, i)
except ValueError as e:
raise MarkdownParseError(str(e), path=path, line=i + 1)
# Value markers are read by ``set_values``, not here; they live in
# prose and attach to nothing
if annotation is not None and annotation[0] not in ("value", "values"):
pending = annotation
pending_line = i + 1
# Whatever a comment contains is commented out, fences included,
# so skip past the whole of one whether it was a directive or not
i = max(end, i + 1)
if pending is not None:
raise MarkdownParseError(
"A Calkit directive comment must be followed by a fenced code "
"block or a bulleted list",
path=path,
line=pending_line,
)
return blocks
def _merge(
pending: tuple[DirectiveKind, dict[str, Any]] | None,
annotation: tuple[DirectiveKind, dict[str, Any]] | None,
) -> tuple[DirectiveKind, dict[str, Any]]:
"""Combine a preceding comment directive with a fence's own annotation.
The comment exists to carry attributes too long for an info string, so
the two are additive; a key given twice is an authoring mistake rather
than something to silently resolve.
"""
if pending is None:
assert annotation is not None
return annotation
if annotation is None:
return pending
if pending[0] != annotation[0]:
raise ValueError(
f"Directive comment declares a '{pending[0]}' but the code "
f"block declares a '{annotation[0]}'"
)
attrs = dict(pending[1])
for key, value in annotation[1].items():
if key in attrs:
raise ValueError(
f"Attribute '{key}' is set in both the directive comment "
"and the code block"
)
attrs[key] = value
return pending[0], attrs
# Parsed blocks, keyed by path and what the file looked like when read.
# Project info is expanded several times over a single status---once to
# check environments, once to compile, once for the status itself---and
# parsing dominates that work, so the same unchanged file would otherwise
# be read and scanned each time.
_PARSE_CACHE: dict[str, tuple[tuple[int, int], list[MarkdownBlock]]] = {}
def parse_markdown_file(path: str) -> list[MarkdownBlock]:
"""Extract Calkit-annotated blocks from a Markdown file.
The returned blocks are shared between callers, so they must be read
rather than modified.
"""
key = Path(path).as_posix()
try:
stat = os.stat(path)
stamp = (stat.st_mtime_ns, stat.st_size)
except OSError:
stamp = None
if stamp is not None:
cached = _PARSE_CACHE.get(key)
if cached is not None and cached[0] == stamp:
return cached[1]
with open(path, encoding="utf-8") as f:
blocks = parse_markdown(f.read(), path=key)
if stamp is not None:
_PARSE_CACHE[key] = (stamp, blocks)
return blocks
# Separator between a Markdown stage's own name and the name of a stage it
# declares, e.g. ``README.md/example``. DVC reserves ``@`` for the names it
# generates from a matrix, and a hand-written one is unaddressable: even
# ``dvc stage list`` refuses the file. ``:`` is no good either, since DVC
# splits a target on the first colon to find the dvc.yaml it belongs to.
STAGE_NAME_SEPARATOR = "/"
# Fence languages Calkit knows how to run, and the extension its extracted
# script needs so the ordinary script-stage machinery can execute it.
LANGUAGE_EXTENSIONS: dict[str, str] = {
"python": ".py",
"py": ".py",
"r": ".R",
"julia": ".jl",
"jl": ".jl",
"sh": ".sh",
"bash": ".sh",
"zsh": ".sh",
"matlab": ".m",
"octave": ".m",
}
# The stage kind each language compiles to. A Markdown stage is extracted
# to a real script, so it reuses the existing script stages rather than
# needing a runtime of its own.
LANGUAGE_STAGE_KINDS: dict[str, str] = {
".py": "python-script",
".R": "r-script",
".jl": "julia-script",
".sh": "shell-script",
".m": "matlab-script",
}
@dataclass
class MarkdownStageSpec:
"""One stage declared across a Markdown file's annotated blocks."""
name: str
language: str
attrs: dict[str, Any]
content: str
script_path: str
# 1-indexed line of the stage's first block, for error messages
line: int = 0
@property
def stage_kind(self) -> str:
return LANGUAGE_STAGE_KINDS[LANGUAGE_EXTENSIONS[self.language]]
def get_stage_script_path(
markdown_path: str, stage_name: str, language: str, as_posix: bool = True
) -> str:
"""Return the path of the script extracted for a Markdown stage.
Stages depend on this derived file rather than on the Markdown itself,
so editing prose doesn't invalidate anything and editing a code block
invalidates only the stage that block belongs to.
"""
ext = LANGUAGE_EXTENSIONS[language]
md_dir = os.path.dirname(markdown_path)
# The extension is kept, so 'README.md' and 'README.markdown' don't
# both derive into a directory named 'README'
md_name = os.path.basename(markdown_path)
p = os.path.join(".calkit", "markdown", md_dir, md_name, stage_name + ext)
if as_posix:
p = Path(p).as_posix()
return p
def get_output_cache_path(
markdown_path: str, stage_name: str, as_posix: bool = True
) -> str:
"""Return the path caching what a stage's output block claims.
Stages depend on this rather than on it being an output, because
output is injected after the pipeline has already run---nothing the
stage command itself writes could satisfy an output declaration. As a
dependency, an output block edited by hand makes its stage stale,
which is what stops the file from quietly claiming something untrue.
"""
md_dir = os.path.dirname(markdown_path)
md_name = os.path.basename(markdown_path)
p = os.path.join(
".calkit", "markdown", "outputs", md_dir, md_name, stage_name + ".txt"
)
if as_posix:
p = Path(p).as_posix()
return p
def extract_outputs(
blocks: list[MarkdownBlock], markdown_path: str
) -> dict[str, str]:
"""Collect the output blocks in a Markdown file, keyed by stage name."""
path = Path(markdown_path).as_posix()
outputs: dict[str, str] = {}
for block in blocks:
if block.kind != "output":
continue
# 'stage' reads better on an output block than 'name' would, since
# the block is not itself the stage
name = block.attrs.get("stage") or block.attrs.get("name")
if not name:
raise MarkdownParseError(
"An output block must name the stage it belongs to, e.g. "
"'calkit output stage=my-stage'",
path=path,
line=block.line,
)
if str(name) in outputs:
raise MarkdownParseError(
f"Stage '{name}' has more than one output block",
path=path,
line=block.line,
)
outputs[str(name)] = block.content
return outputs
def write_output_caches(
outputs: dict[str, str], markdown_path: str, wdir: str | None = None
) -> dict[str, str]:
"""Write each output block's content to its cache file.
Returns a map of stage name -> cache path.
"""
paths = {}
for stage_name, content in outputs.items():
rel = get_output_cache_path(markdown_path, stage_name)
paths[stage_name] = rel
fpath = os.path.join(wdir, rel) if wdir is not None else rel
text = (
content
if content.endswith("\n") or not content
else (content + "\n")
)
if os.path.isfile(fpath):
with open(fpath, encoding="utf-8") as f:
if f.read() == text:
continue
os.makedirs(os.path.dirname(fpath), exist_ok=True)
with open(fpath, "w", encoding="utf-8", newline="\n") as f:
f.write(text)
return paths
def prune_derived_files(
markdown_path: str,
stage_names: list[str],
output_stage_names: list[str],
wdir: str | None = None,
) -> list[str]:
"""Delete derived files for stages the Markdown no longer declares.
Renaming or removing a stage would otherwise leave its extracted
script and cached output behind, where they look like part of the
project.
"""
md_dir = os.path.dirname(markdown_path)
md_name = os.path.basename(markdown_path)
expected = {
os.path.join(".calkit", "markdown", md_dir, md_name): {
name + ext
for name in stage_names
for ext in set(LANGUAGE_EXTENSIONS.values())
},
os.path.join(".calkit", "markdown", "outputs", md_dir, md_name): {
name + ".txt" for name in output_stage_names
},
}
removed = []
for rel_dir, keep in expected.items():
dpath = os.path.join(wdir, rel_dir) if wdir is not None else rel_dir
if not os.path.isdir(dpath):
continue
for fname in sorted(os.listdir(dpath)):
fpath = os.path.join(dpath, fname)
if not os.path.isfile(fpath) or fname in keep:
continue
os.remove(fpath)
removed.append(Path(os.path.join(rel_dir, fname)).as_posix())
return removed
def extract_stages(
blocks: list[MarkdownBlock], markdown_path: str
) -> dict[str, MarkdownStageSpec]:
"""Group annotated blocks into stages, concatenating by name.
Blocks sharing a name join in document order, which is what lets one
example be narrated across several code blocks and still run as a
single script.
"""
path = Path(markdown_path).as_posix()
specs: dict[str, MarkdownStageSpec] = {}
for block in blocks:
if block.kind != "stage":
continue
name = block.name
if not name:
raise MarkdownParseError(
"A stage block must declare a name", path=path, line=block.line
)
check_name(name, "Stage", path, block.line)
if block.language is None:
raise MarkdownParseError(
f"Stage '{name}' has a code block with no language, so "
"Calkit can't tell how to run it",
path=path,
line=block.line,
)
language = block.language.lower()
if language not in LANGUAGE_EXTENSIONS:
raise MarkdownParseError(
f"Stage '{name}' uses language '{block.language}', which "
"Calkit can't run; supported languages are "
f"{', '.join(sorted(set(LANGUAGE_EXTENSIONS)))}",
path=path,
line=block.line,
)
attrs = {k: v for k, v in block.attrs.items() if k != "name"}
spec = specs.get(name)
if spec is None:
specs[name] = MarkdownStageSpec(
name=name,
language=language,
attrs=attrs,
content=block.content,
script_path=get_stage_script_path(path, name, language),
line=block.line,
)
continue
if language != spec.language:
raise MarkdownParseError(
f"Stage '{name}' mixes languages '{spec.language}' and "
f"'{language}'; a stage becomes one script, so its blocks "
"must share a language",
path=path,
line=block.line,
)
for key, value in attrs.items():
if key in spec.attrs and spec.attrs[key] != value:
raise MarkdownParseError(
f"Stage '{name}' sets attribute '{key}' to conflicting "
f"values ({spec.attrs[key]!r} and {value!r})",
path=path,
line=block.line,
)
spec.attrs[key] = value
spec.content += "\n\n" + block.content
return specs
def write_stage_scripts(
specs: dict[str, MarkdownStageSpec], wdir: str | None = None
) -> list[str]:
"""Write out each stage's extracted script, returning changed paths.
Only files whose content actually differs are rewritten, since these
are stage dependencies and a needless touch is a needless rerun.
"""
changed = []
for spec in specs.values():
fpath = spec.script_path
if wdir is not None:
fpath = os.path.join(wdir, fpath)
content = spec.content
if not content.endswith("\n"):
content += "\n"
if os.path.isfile(fpath):
with open(fpath, encoding="utf-8") as f:
if f.read() == content:
continue
os.makedirs(os.path.dirname(fpath), exist_ok=True)
with open(fpath, "w", encoding="utf-8", newline="\n") as f:
f.write(content)
changed.append(spec.script_path)
return changed
def get_stage_names(markdown_path: str, stage_name: str) -> list[str]:
"""Return the names of the pipeline stages a Markdown file declares.
Used to resolve a target naming the file as a whole (``calkit run
README.md``) into the stages it stands for, since those are what DVC
knows about.
"""
blocks = parse_markdown_file(markdown_path)
specs = extract_stages(blocks, Path(markdown_path).as_posix())
return [stage_name + STAGE_NAME_SEPARATOR + name for name in specs]
def get_markdown_stages(ck_info: dict) -> dict[str, str]:
"""Map each markdown stage's name to the Markdown file it reads.
A stage is addressed by its name, like any other kind; ``calkit xr``
keys the stage by the file's path, which is what makes ``calkit run
README.md`` work.
"""
stages = ck_info.get("pipeline", {}).get("stages", {})
if not isinstance(stages, dict):
return {}
return {
name: Path(str(cfg["target_path"])).as_posix()
for name, cfg in stages.items()
if isinstance(cfg, dict)
and cfg.get("kind") == "markdown"
# Not a valid stage otherwise; expansion reports that properly
and cfg.get("target_path")
}
class MarkdownDocument(BaseModel):
"""What a Markdown file declares, read in one pass."""
path: str
text: str
blocks: list[MarkdownBlock]
# Stage name -> the stage its blocks declare
stages: dict[str, MarkdownStageSpec]
# Stage name -> what its output block claims it printed
outputs: dict[str, str]
def read_markdown(text: str, path: str) -> MarkdownDocument:
"""Read Markdown text into the stages and outputs it declares.
Raises ``MarkdownParseError`` for an annotation that can't be read.
"""
path = Path(path).as_posix()
blocks = parse_markdown(text, path=path)
return MarkdownDocument(
path=path,
text=text,
blocks=blocks,
stages=extract_stages(blocks, path),
outputs=extract_outputs(blocks, path),
)
class MarkdownEnvironments(BaseModel):
"""The environments a Markdown file's stages run in.
Declared environments are those the file annotates; detected ones are
built from the imports of stages that name none, and ``assignments``
says which detected environment each such stage was given.
"""
declared: dict[str, dict]
detected: dict[str, dict]
assignments: dict[str, str]
@property
def detected_public(self) -> dict[str, dict]:
"""Detected environments as they would be written to calkit.yaml."""
return {
name: {k: v for k, v in env.items() if not k.startswith("_")}
for name, env in self.detected.items()
}
def get_environments(
doc: MarkdownDocument,
existing_env_names: list[str] | None = None,
default_env: str | None = None,
) -> MarkdownEnvironments:
"""Collect a file's declared environments and detect any it still needs.
A detected environment is named to avoid both the project's existing
environments and the ones the file declares.
"""
declared = extract_environments(doc.blocks, doc.path)
detected, assignments = detect_environments(
doc.stages,
doc.path,
existing_env_names=list(existing_env_names or []) + list(declared),
default_env=default_env,
)
return MarkdownEnvironments(
declared=declared, detected=detected, assignments=assignments
)
# Environment kinds that can be declared in Markdown, and the spec file
# each one is written to. A list of packages means something different to
# each toolchain, so the kind decides how the list is rendered.
ENV_SPEC_FILENAMES: dict[str, str] = {
"uv-venv": "requirements.txt",
"uv": "pyproject.toml",
"conda": "environment.yml",
"renv": "DESCRIPTION",
"julia": "Project.toml",
}
# Which environment kind a stage's language implies, so a Markdown file
# declaring one environment for R or Julia code needn't spell it out.
LANGUAGE_ENV_KINDS: dict[str, str] = {
# uv over uv-venv for Python: near enough the same interface, with a
# better lock file
"python": "uv",
"py": "uv",
"r": "renv",
"julia": "julia",
"jl": "julia",
}
_ENV_SOURCE_RE = re.compile(r"^(?:Generated from|Declared in) (?P<path>\S+)\.")
def env_source_path(env: dict) -> str | None:
"""Return the Markdown file an environment entry was generated from.
The description written with a generated entry is the record of
where it came from, which is what lets a later compile tell one of
these from an environment the user wrote.
"""
description = env.get("description") if isinstance(env, dict) else None
if not isinstance(description, str):
return None
m = _ENV_SOURCE_RE.match(description)
return None if m is None else m.group("path")
def _env_definition(env: dict) -> dict:
"""Return the parts of an environment that define it.
The description is a generated annotation saying where the
environment came from, so two entries differing only there are the
same environment.
"""
return {k: v for k, v in dict(env).items() if k != "description"}
def get_env_spec_path(
env_name: str, kind: str = "uv-venv", as_posix: bool = True
) -> str:
"""Return the path of the spec file written for a Markdown environment.
This lives alongside the specs Calkit already generates when it
detects an environment for a stage, so the environment is an ordinary
one by the time anything else looks at it.
"""
p = os.path.join(".calkit", "envs", env_name, ENV_SPEC_FILENAMES[kind])
if as_posix:
p = Path(p).as_posix()
return p
def _rel_root_from_spec(spec_path: str) -> str:
"""Path of the project root, relative to a spec file's directory."""
return Path(os.path.relpath(".", os.path.dirname(spec_path))).as_posix()
def render_env_spec(
kind: str,
content: str,
env_name: str,
python: str | None = None,
local_package: tuple[str | None, str] | None = None,
wdir: str | None = None,
) -> str:
"""Render a Markdown environment block into its spec file's content.
A fenced block is taken verbatim, since that's the escape hatch for
anything the list form can't say. A bulleted list is a list of
package names, which each toolchain spells differently, so it's
rendered through the same builders Calkit uses when it detects
dependencies itself.
``local_package`` is ``(name, relative path to the project root)``
when the environment installs the project's own package alongside
other things, in which case the package comes from the working tree
rather than a registry---read inside the repo, that is what the
install command means.
"""
import calkit.environments
if kind == "uv-venv":
spec = content
if local_package is not None:
spec = spec.rstrip("\n") + f"\n-e {local_package[1]}\n"
return spec
deps = [line.strip() for line in content.splitlines() if line.strip()]
if kind == "uv":
kwargs = {} if python is None else {"python_version": python}
all_deps = deps
if local_package is not None and local_package[0] is not None:
all_deps = deps + [local_package[0]]
spec = calkit.environments.create_uv_pyproject_content(
all_deps, project_name=env_name, **kwargs
)
if local_package is not None and local_package[0] is not None:
spec += (
"\n[tool.uv.sources]\n"
f"{local_package[0]} = {{ path = "
f'"{local_package[1]}", editable = true }}\n'
)
return spec
if kind == "conda":
spec = calkit.environments.create_conda_environment_content(
deps, project_name=env_name
)
if local_package is not None:
spec += f' - pip\n - pip:\n - "-e {local_package[1]}"\n'
return spec
if kind == "renv":
all_deps = deps
if local_package is not None and local_package[0] is not None:
all_deps = deps + [local_package[0]]
spec = calkit.environments.create_r_description_content(
all_deps, project_name=env_name
)
if local_package is not None:
spec += f"Remotes: local::{local_package[1]}\n"
return spec
if kind == "julia":
spec = calkit.environments.create_julia_project_file_content(
deps, project_name=env_name
)
if local_package is not None and local_package[0] is not None:
uuid = local_package_uuid(wdir=wdir)
if "[deps]" not in spec:
spec += "\n[deps]\n"
if uuid is not None:
spec = spec.rstrip("\n") + (
f'\n{local_package[0]} = "{uuid}"\n'
)
spec += (
f"\n[sources]\n{local_package[0]} = "
f'{{path = "{local_package[1]}"}}\n'
)
return spec
raise ValueError(f"Unsupported environment kind: {kind}")
def extract_environments(
blocks: list[MarkdownBlock], markdown_path: str
) -> dict[str, dict]:
"""Collect the environment declarations in a Markdown file.
The result is deliberately unfinished: an environment's kind can be
left out and inferred from the language of the stages that use it, so
the spec path and content aren't known until the stages are read too.
Call ``resolve_environments`` to finish them.
"""
path = Path(markdown_path).as_posix()
envs: dict[str, dict] = {}
for block in blocks:
if block.kind != "environment":
continue
name = block.name
if not name:
raise MarkdownParseError(
"An environment block must declare a name",
path=path,
line=block.line,
)
check_name(name, "Environment", path, block.line)
install = (
parse_install_block(block.content, block.language)
if block.source == "fence"
else None
)
if name in envs:
# Two blocks of install commands under one name are additive:
# a README often installs in stages, or shows the same package
# fetched two ways, and both describe one environment.
previous = envs[name].get("_install")
if install is None or previous is None:
raise MarkdownParseError(
f"Environment '{name}' is declared more than once",
path=path,
line=block.line,
)
if install.kind == previous.kind:
install = merge_install_specs([previous, install])
else:
install = previous
envs[name]["_install"] = install
envs[name]["_content"] = "\n".join(install.packages)
continue
attrs = {k: v for k, v in block.attrs.items() if k != "name"}
kind = attrs.pop("kind", None)
if kind is not None and str(kind) not in ENV_SPEC_FILENAMES:
raise MarkdownParseError(
f"Environment '{name}' has kind '{kind}'; environments "
"declared in Markdown must be one of "
f"{', '.join(sorted(ENV_SPEC_FILENAMES))}",
path=path,
line=block.line,
)
env: dict = {"_kind": None if kind is None else str(kind)}
python = attrs.pop("python", None)
if python is not None:
env["python"] = str(python)
julia = attrs.pop("julia", None)
if julia is not None:
env["_julia"] = str(julia)
description = attrs.pop("description", None)
if description is not None:
env["description"] = str(description)
if attrs:
raise MarkdownParseError(
f"Environment '{name}' has unrecognized attribute(s): "
f"{', '.join(sorted(attrs))}",
path=path,
line=block.line,
)
# A fence is the escape hatch for anything the list form can't
# say, so it's kept verbatim whatever the kind.
env["_verbatim"] = block.source == "fence"
env["_content"] = block.content
env["_line"] = block.line
# ...unless the fence is a command that installs the packages,
# which is how a README normally says what its code needs. That
# states the same thing imperatively, so it's read as a package
# list, and the installer chosen says which kind of environment
# it is.
if install is not None:
env["_install"] = install
env["_verbatim"] = False
env["_content"] = "\n".join(install.packages)
if kind is None:
env["_kind"] = install.kind
envs[name] = env
return envs
def resolve_environments(
envs: dict[str, dict],
specs: dict[str, MarkdownStageSpec],
markdown_path: str,
default_env: str | None = None,
wdir: str | None = None,
) -> dict[str, dict]:
"""Finish environment declarations, inferring kinds where they're absent.
An R or Julia README shouldn't have to name its environment's kind
when the code blocks using it already say what language they are.
"""
path = Path(markdown_path).as_posix()
# Which languages each environment is used by
langs_by_env: dict[str, set[str]] = {}
for spec in specs.values():
env_name = spec.attrs.get("environment", default_env)
if env_name is None:
continue
langs_by_env.setdefault(str(env_name), set()).add(spec.language)
for name, env in envs.items():
kind = env["_kind"]
if kind is None:
langs = langs_by_env.get(name, set())
kinds = {
LANGUAGE_ENV_KINDS[lang]
for lang in langs
if lang in LANGUAGE_ENV_KINDS
}
if len(kinds) > 1:
raise MarkdownParseError(
f"Environment '{name}' is used by stages in more than "
f"one language ({', '.join(sorted(langs))}); give it an "
"explicit kind",
path=path,
line=env["_line"],
)
kind = kinds.pop() if kinds else "uv"
env["_kind"] = kind
python = env.pop("python", None)
if python is not None and kind not in ("uv-venv", "uv"):
raise MarkdownParseError(
f"Environment '{name}' is a '{kind}' environment, which "
"takes no 'python' version",
path=path,
line=env["_line"],
)
# A uv environment has no 'python' field; its version is declared
# by the pyproject.toml the spec is rendered into.
if python is not None and kind == "uv-venv":
env["python"] = python
if python is not None and kind == "uv":
env["_python"] = python
julia = env.pop("_julia", None)
if julia is not None and kind != "julia":
raise MarkdownParseError(
f"Environment '{name}' is a '{kind}' environment, which "
"takes no 'julia' version",
path=path,
line=env["_line"],
)
if kind == "julia":
if julia is None:
raise MarkdownParseError(
f"Environment '{name}' is a Julia environment, which "
"needs a version, e.g. 'calkit environment "
f"name={name} julia=1.12'",
path=path,
line=env["_line"],
)
env["julia"] = julia
install = env.pop("_install", None)
# A dev install says the environment includes this project itself.
# Which other packages it names decides the shape: none, and the
# project's own spec file already describes the environment, so
# point at that; some, and a spec has to be generated for them,
# with the project's package coming from the working tree rather
# than a registry---read inside the repo, that is what the
# install command means.
local_pkg = (
local_package_name(install.language, wdir=wdir)
if install is not None
else None
)
local_key = None if local_pkg is None else _requirement_key(local_pkg)
others = [
p
for p in (install.packages if install is not None else [])
if _requirement_key(p) != local_key
]
is_dev = install is not None and (
install.dev
or (local_key is not None and len(others) < len(install.packages))
)
dev_spec = resolve_dev_install(install, wdir=wdir) if is_dev else None
# The generated form needs the package's name to refer to it; a
# project without one (a bare requirements.txt, say) can still be
# named by path in a requirements file
local_dev = (
is_dev
and others
and dev_spec is not None
and (local_pkg is not None or kind == "uv-venv")
)
if local_dev:
env["path"] = get_env_spec_path(name, kind=kind)
env["_spec_content"] = render_env_spec(
kind,
"\n".join(others),
name,
python=python,
local_package=(local_pkg, _rel_root_from_spec(env["path"])),
wdir=wdir,
)
env["kind"] = kind
if python is not None and kind == "uv":
env["_python"] = python
env.setdefault(
"description",
f"Generated from {path}. "
"Changes made here will be overwritten.",
)
continue
if dev_spec is not None:
kind, env["kind"] = dev_spec[0], dev_spec[0]
env["path"] = dev_spec[1]
elif install is not None and install.spec_path:
# ...as does a command naming a spec file, when it is there
spec_path = install.spec_path
full = os.path.join(wdir, spec_path) if wdir else spec_path
if os.path.isfile(full):
env["path"] = Path(spec_path).as_posix()
else:
env["path"] = get_env_spec_path(name, kind=kind)
else:
env["path"] = get_env_spec_path(name, kind=kind)
env["kind"] = kind
if dev_spec is not None or env["path"] != get_env_spec_path(
name, kind=kind
):
# The spec is a file the project already keeps, so leave it be
env["_spec_content"] = None
env.setdefault("description", f"Declared in {path}.")
elif env["_verbatim"]:
env["_spec_content"] = env["_content"]
else:
env["_spec_content"] = render_env_spec(
kind, env["_content"], name, python=python
)
if env["_spec_content"] is not None:
# Say where this came from in the environment itself rather
# than a YAML comment, which a user could delete without the
# notice being restored. This mirrors the 'desc' Calkit
# writes on DVC stages.
env.setdefault(
"description",
f"Generated from {path}. "
"Changes made here will be overwritten.",
)
return envs
def write_env_specs(envs: dict[str, dict], wdir: str | None = None) -> None:
"""Write each Markdown environment's spec file, if it changed.
The spec is derived, but it is what ``uv`` reads and what the lock is
resolved from, so it must be byte-stable or the lock churns on every
compile.
"""
for env in envs.values():
# An environment that points at a spec file already in the project
# -- what a dev install declares -- has nothing to generate
if env.get("_spec_content") is None:
continue
# uv resolves its interpreter from a .python-version in the project
# directory. Without one the version only floats above the
# requires-python floor, so the environment isn't really pinned.
python = env.get("_python")
if python is not None:
pv_path = os.path.join(
os.path.dirname(env["path"]), ".python-version"
)
if wdir is not None:
pv_path = os.path.join(wdir, pv_path)
pv_content = str(python) + "\n"
if not os.path.isfile(pv_path) or (
open(pv_path, encoding="utf-8").read() != pv_content
):
os.makedirs(os.path.dirname(pv_path), exist_ok=True)
with open(pv_path, "w", encoding="utf-8", newline="\n") as f:
f.write(pv_content)
content = env["_spec_content"]
if not content.endswith("\n"):
content += "\n"
fpath = env["path"]
if wdir is not None:
fpath = os.path.join(wdir, fpath)
if os.path.isfile(fpath):
with open(fpath, encoding="utf-8") as f:
if f.read() == content:
continue
os.makedirs(os.path.dirname(fpath), exist_ok=True)
with open(fpath, "w", encoding="utf-8", newline="\n") as f:
f.write(content)
@dataclass
class MarkdownExpansion:
"""What a project's Markdown files contributed to its pipeline."""
ck_info: dict
# The Markdown files that were expanded, so their previously generated
# entries can be told apart from a user's own
markdown_paths: list[str] = field(default_factory=list)
script_paths: list[str] = field(default_factory=list)
# Environment name -> definition, and the file that declared it
environments: dict[str, dict] = field(default_factory=dict)
environment_sources: dict[str, str] = field(default_factory=dict)
# Pipeline stage name -> path caching that stage's output block
output_cache_paths: dict[str, str] = field(default_factory=dict)
# Derived files deleted because the Markdown no longer declares them
removed_paths: list[str] = field(default_factory=list)
def expand_ck_info(
ck_info: dict, wdir: str | None = None
) -> MarkdownExpansion:
"""Return project info with markdown stages replaced by what they declare.
A markdown stage stands in for however many stages and environments
its file declares. Expanding here rather than at compile time means
everything that reads project info---environment checking, lock file
paths, status filtering---sees ordinary stages and environments and
needs no knowledge of Markdown.
The input is not modified, so callers that go on to write
``calkit.yaml`` can't accidentally persist derived entries.
Environments are returned as well as merged, because they have to be
persisted to ``calkit.yaml``: a stage's command runs ``calkit xenv``
as a subprocess, which reads the environment back off disk.
"""
from copy import deepcopy
stages = ck_info.get("pipeline", {}).get("stages", {})
if not isinstance(stages, dict):
return MarkdownExpansion(ck_info=ck_info)
md_stage_names = [
name
for name, cfg in stages.items()
if isinstance(cfg, dict) and cfg.get("kind") == "markdown"
]
if not md_stage_names:
return MarkdownExpansion(ck_info=ck_info)
out = deepcopy(ck_info)
out_stages = out["pipeline"]["stages"]
out_envs = out.setdefault("environments", {})
if not isinstance(out_envs, dict):
raise ValueError("Project 'environments' must be a mapping")
result = MarkdownExpansion(ck_info=out)
for stage_name in md_stage_names:
md_cfg = out_stages.pop(stage_name)
# Markdown stages are removed here, before the pipeline is
# validated, so nothing else would catch a typo'd key in one.
from calkit.models.pipeline import MarkdownStage
try:
md_stage = MarkdownStage.model_validate(
dict(md_cfg) | {"name": stage_name}
)
except Exception as e:
raise ValueError(
f"Markdown stage '{stage_name}' is not defined properly: {e}"
)
md_path = md_stage.markdown_path
if md_path in result.markdown_paths:
raise ValueError(
f"Markdown stage '{stage_name}' reads '{md_path}', which "
"another markdown stage already does"
)
result.markdown_paths.append(md_path)
read_path = os.path.join(wdir, md_path) if wdir else md_path
if not os.path.isfile(read_path):
raise ValueError(
f"Markdown stage '{stage_name}' points at '{md_path}', "
"which does not exist"
)
blocks = parse_markdown_file(read_path)
envs = extract_environments(blocks, md_path)
specs = extract_stages(blocks, md_path)
if not specs:
raise ValueError(
f"Markdown stage '{stage_name}' declares no stages; "
"annotate a code block with 'calkit stage name=<name>' "
"to define one"
)
# A file declaring exactly one environment shouldn't have to name
# it on every block.
default_env = md_cfg.get("environment")
if default_env is None and len(envs) == 1:
default_env = next(iter(envs))
# Kinds can be inferred from the languages of the stages using
# each environment, so this needs both halves read first.
envs = resolve_environments(
envs, specs, md_path, default_env=default_env, wdir=wdir
)
for env_name, env in envs.items():
public = {k: v for k, v in env.items() if not k.startswith("_")}
# Two files can't both own one environment, however alike
# their entries look: the spec file is written from whichever
# was read last, and the other file's stages would run in it.
other = result.environment_sources.get(env_name)
if other is not None:
raise ValueError(
f"Environment '{env_name}' is declared in both "
f"'{other}' and '{md_path}'; it can only be defined in "
"one place"
)
existing = out_envs.get(env_name)
# An identical entry is this same definition written back on
# an earlier compile, not a competing one.
if existing is not None and _env_definition(
existing
) != _env_definition(public):
raise ValueError(
f"Environment '{env_name}' is declared in '{md_path}' "
"and differently in calkit.yaml; it can only be defined "
"in one place"
)
out_envs[env_name] = public
result.environments[env_name] = public
result.environment_sources[env_name] = md_path
write_env_specs(envs, wdir=wdir)
write_stage_scripts(specs, wdir=wdir)
result.script_paths += [s.script_path for s in specs.values()]
# An output block is cached to a file the stage depends on, so
# editing the block by hand makes the stage stale rather than
# leaving the file quietly claiming something untrue.
outputs = extract_outputs(blocks, md_path)
unknown = set(outputs) - set(specs)
if unknown:
raise ValueError(
f"{md_path} has output block(s) for stage(s) that it does "
f"not declare: {', '.join(sorted(unknown))}"
)
output_cache_paths = write_output_caches(outputs, md_path, wdir=wdir)
result.output_cache_paths.update(
{
stage_name + STAGE_NAME_SEPARATOR + name: path
for name, path in output_cache_paths.items()
}
)
result.removed_paths += prune_derived_files(
md_path,
stage_names=list(specs),
output_stage_names=list(outputs),
wdir=wdir,
)
for spec in specs.values():
sub_name = stage_name + STAGE_NAME_SEPARATOR + spec.name
if sub_name in out_stages:
raise ValueError(
f"Stage '{sub_name}' from '{md_path}' conflicts with an "
"existing pipeline stage"
)
sub: dict = {
"kind": spec.stage_kind,
"script_path": spec.script_path,
}
# What the file declares for itself applies to every stage in
# it; a block's own attributes then take precedence
for key in ("always_run", "frozen", "scheduler"):
if key in md_cfg and md_cfg[key] is not None:
sub[key] = md_cfg[key]
sub.update(spec.attrs)
inputs = list(md_cfg.get("inputs") or []) + [
i
for i in spec.attrs.get("inputs") or []
if i not in (md_cfg.get("inputs") or [])
]
cache_path = output_cache_paths.get(spec.name)
if cache_path is not None:
inputs.append(cache_path)
if inputs:
sub["inputs"] = inputs
if "environment" not in sub:
sub["environment"] = default_env or "_system"
if spec.stage_kind == "shell-script" and "shell" not in sub:
sub["shell"] = "sh" if spec.language == "sh" else spec.language
out_stages[sub_name] = sub
return result
# Languages whose dependencies Calkit can read out of the code itself.
# Shell and MATLAB have no import statements to go on, so stages in those
# languages keep whatever environment they were given.
# Fence languages whose content is a shell session rather than a program.
SHELL_LANGUAGES = frozenset(
{"sh", "bash", "zsh", "shell", "console", "shell-session", "terminal"}
)
# Which environment kind each installer produces, and the language whose
# stages it therefore serves.
INSTALLER_ENV_KINDS: dict[str, str] = {
"pip": "uv-venv",
"uv-pip": "uv-venv",
"uv": "uv",
"conda": "conda",
"julia": "julia",
"r": "renv",
}
ENV_KIND_LANGUAGES: dict[str, str] = {
"uv-venv": "python",
"uv": "python",
"conda": "python",
"julia": "julia",
"renv": "r",
}
# The spec file a dev install resolves to, in the order a language's
# toolchains are preferred when more than one is present.
DEV_INSTALL_SPECS: dict[str, list[tuple[str, str]]] = {
"python": [
("uv", "pyproject.toml"),
("uv-venv", "requirements.txt"),
("conda", "environment.yml"),
],
"julia": [("julia", "Project.toml")],
"r": [("renv", "DESCRIPTION")],
}
# Options that consume the token after them, so their value isn't mistaken
# for a package name---``conda install -c conda-forge numpy`` asks for one
# package, not two.
_VALUE_OPTIONS = frozenset(
{
"-c",
"--channel",
"-n",
"--name",
"-p",
"--prefix",
"-f",
"--file",
"-r",
"--requirement",
"--python",
"--index-url",
"--extra-index-url",
"--find-links",
"--group",
"--target",
"--constraint",
}
)
_SPEC_FILE_OPTIONS = frozenset({"-r", "--requirement", "-f", "--file"})
@dataclass
class InstallSpec:
"""What a block of install commands asks for."""
kind: str
packages: list[str] = field(default_factory=list)
# Set when the command installs the project in the working directory
# (``pip install -e .``, ``Pkg.develop(path=".")``, ``uv sync``), which
# is an environment already described by a spec file on disk
dev: bool = False
# Set when the command names a spec file, as ``pip install -r`` does
spec_path: str | None = None
@property
def language(self) -> str:
return ENV_KIND_LANGUAGES[self.kind]
def _requirement_key(requirement: str) -> str:
"""Reduce a requirement to the package it names, for de-duplication.
A README often shows the same install more than once---over SSH and
over HTTPS, say---and those are alternatives, not two dependencies.
"""
req = requirement.strip()
# PEP 508 names the package before the URL it comes from
m = re.match(r"^(?P<name>[A-Za-z0-9._-]+)\s*@\s*\S+$", req)
if m is not None:
return _normalize_package_name(m.group("name"))
if "://" in req:
name = req.rstrip("/").rsplit("/", 1)[-1]
name = name.split("#")[0].split("?")[0]
return _normalize_package_name(name.removesuffix(".git"))
for sep in ["===", "==", ">=", "<=", "~=", "!=", ">", "<", "[", ";", "="]:
req = req.split(sep)[0]
return _normalize_package_name(req)
def _normalize_package_name(name: str) -> str:
"""Normalize a package name the way a package index would."""
return re.sub(r"[-_.]+", "-", name.strip()).lower()
# The prompts a session transcript writes before what was typed. ``pkg>``
# is missing on purpose: it marks Julia's package mode, so it is read
# rather than stripped.
_PROMPT_RE = re.compile(r"^(?:\$|%|>>>|\.\.\.|julia>|help\?>|shell>)\s+")
# What marks a block as a transcript of a session rather than a script:
# its output is interleaved with its input, so it can't be run as it
# stands, and Calkit leaves it alone rather than guessing which is which.
_TRANSCRIPT_RE = re.compile(r"^(?:>>>|\.\.\.|julia>|pkg>|help\?>|In \[\d+\]:)")
def is_repl_transcript(content: str) -> bool:
"""Determine whether a block is a session transcript, not a script."""
return any(
_TRANSCRIPT_RE.match(line.strip()) for line in content.splitlines()
)
def _strip_prompt(line: str) -> str:
"""Drop the prompt a session transcript writes before what was typed."""
stripped = line.strip()
return _PROMPT_RE.sub("", stripped).strip()
def _parse_shell_install(line: str) -> InstallSpec | None:
"""Read one shell command as an install, or return None if it isn't."""
import shlex
try:
tokens = shlex.split(line, comments=True)
except ValueError:
return None
while tokens and tokens[0] in ("sudo", "!"):
tokens = tokens[1:]
if not tokens:
return None
installer: str | None = None
dev = False
rest: list[str] = []
if tokens[0] in ("pip", "pip3") and tokens[1:2] == ["install"]:
installer, rest = "pip", tokens[2:]
elif tokens[:4] == ["python", "-m", "pip", "install"]:
installer, rest = "pip", tokens[4:]
elif tokens[0] == "uv" and tokens[1:2] == ["add"]:
installer, rest = "uv", tokens[2:]
elif tokens[:3] == ["uv", "pip", "install"]:
installer, rest = "uv-pip", tokens[3:]
elif tokens[0] == "uv" and tokens[1:2] in (["sync"], ["lock"]):
# These operate on the project in the working directory, so what
# they declare is that project's own environment
installer, dev, rest = "uv", True, []
elif tokens[0] in ("conda", "mamba", "micromamba"):
if tokens[1:2] == ["install"]:
installer, rest = "conda", tokens[2:]
elif tokens[1:3] == ["env", "create"]:
installer, rest = "conda", tokens[3:]
elif tokens[1:2] == ["create"]:
installer, rest = "conda", tokens[2:]
else:
return None
if installer is None:
return None
packages: list[str] = []
spec_path: str | None = None
i = 0
while i < len(rest):
arg = rest[i]
if arg.startswith("-"):
flag, _, inline = arg.partition("=")
if inline:
if flag in _SPEC_FILE_OPTIONS:
spec_path = inline
i += 1
continue
if flag in _VALUE_OPTIONS:
if flag in _SPEC_FILE_OPTIONS and i + 1 < len(rest):
spec_path = rest[i + 1]
i += 2
continue
i += 1
continue
if arg in (".", "./") or arg.startswith(("./", "../")):
dev = True
else:
packages.append(arg)
i += 1
if not (packages or dev or spec_path):
return None
return InstallSpec(
kind=INSTALLER_ENV_KINDS[installer],
packages=packages,
dev=dev,
spec_path=spec_path,
)
_JULIA_ADD_RE = re.compile(r"Pkg\.add\.?\((?P<args>.*)\)\s*$", re.DOTALL)
_JULIA_DEV_RE = re.compile(r"Pkg\.(?:develop|dev)\s*\(")
_JULIA_PKG_MODE_RE = re.compile(r"^(?:pkg>|julia>\s*\]|\])\s*(?P<rest>.*)$")
_QUOTED_RE = re.compile(r"""["']([^"']+)["']""")
# Statements that say "use the project here" rather than fetch something
_JULIA_PROJECT_CALLS = ("Pkg.instantiate(", "Pkg.activate(", "Pkg.resolve(")
_JULIA_NOOP_CALLS = ("Pkg.status(", "Pkg.precompile(", "Pkg.update(")
def _parse_julia_install(content: str) -> InstallSpec | None:
"""Read a Julia block as ``Pkg`` calls, or return None if it isn't one."""
packages: list[str] = []
dev = False
saw_install = False
for statement in _install_statements(content, separators=";"):
mode = _JULIA_PKG_MODE_RE.match(statement)
if mode is not None:
rest = mode.group("rest").split()
if rest[:1] == ["add"]:
packages += rest[1:]
saw_install = True
continue
if rest[:1] in (["dev"], ["develop"]):
dev = True
saw_install = True
continue
if rest[:1] in (["instantiate"], ["activate"], ["resolve"]):
dev = True
saw_install = True
continue
return None
if statement in ("using Pkg", "import Pkg"):
continue
if statement.startswith(_JULIA_NOOP_CALLS):
continue
if statement.startswith(_JULIA_PROJECT_CALLS):
dev = True
saw_install = True
continue
if _JULIA_DEV_RE.match(statement):
dev = True
saw_install = True
continue
m = _JULIA_ADD_RE.match(statement)
if m is None:
return None
found = _QUOTED_RE.findall(m.group("args"))
if not found:
return None
packages += found
saw_install = True
if not saw_install:
return None
return InstallSpec(kind="julia", packages=packages, dev=dev)
_R_INSTALL_RE = re.compile(
r"^(?:utils::)?install\.packages\s*\((?P<args>.*)\)\s*$", re.DOTALL
)
_R_REMOTE_RE = re.compile(
r"^(?:remotes|devtools|pak)::(?:install_github|install_gitlab|pkg_install)"
r"\s*\((?P<args>.*)\)\s*$",
re.DOTALL,
)
_R_SELF_RE = re.compile(
r"^(?:renv::(?:restore|init|snapshot|hydrate)|devtools::install|"
r"renv::install)\s*\(\s*\)\s*$"
)
def _parse_r_install(content: str) -> InstallSpec | None:
"""Read an R block as package installs, or return None if it isn't."""
packages: list[str] = []
dev = False
saw_install = False
for statement in _install_statements(content, separators=";"):
if _R_SELF_RE.match(statement):
dev = True
saw_install = True
continue
m = _R_INSTALL_RE.match(statement) or _R_REMOTE_RE.match(statement)
if m is None:
if statement.startswith("renv::install("):
m = re.match(r"^renv::install\((?P<args>.*)\)\s*$", statement)
if m is None:
return None
args = m.group("args")
found = _QUOTED_RE.findall(args)
if not found:
return None
for pkg in found:
# A GitHub remote is written owner/repo; the package is the repo
packages.append(pkg.rsplit("/", 1)[-1])
saw_install = True
if not saw_install:
return None
return InstallSpec(kind="renv", packages=packages, dev=dev)
def _install_statements(content: str, separators: str = "") -> list[str]:
"""Split block content into statements, dropping blanks and comments."""
statements = []
for line in content.splitlines():
line = _strip_prompt(line)
if not line or line.startswith("#"):
continue
parts = [line]
for sep in separators:
parts = [p for part in parts for p in part.split(sep)]
for part in parts:
part = part.strip()
if part and not part.startswith("#"):
statements.append(part)
return statements
def parse_install_block(
content: str, language: str | None
) -> InstallSpec | None:
"""Read a code block as an environment declared by installing it.
A README says what a project needs by showing the command that
installs it, not by listing packages, so those commands are read as
the environment they describe. Every statement in the block has to be
an install for it to count: a block that also does work is code.
"""
if not content.strip():
return None
lang = (language or "").lower()
if lang in ("julia", "jl"):
return _parse_julia_install(content)
if lang == "r":
return _parse_r_install(content)
if lang not in SHELL_LANGUAGES:
return None
specs = []
for statement in _install_statements(content):
spec = _parse_shell_install(statement)
if spec is None:
return None
specs.append(spec)
if not specs:
return None
# A block that mixes installers---pip then conda---is describing
# alternatives rather than one environment, so the first wins
kind = specs[0].kind
merged = InstallSpec(kind=kind)
for spec in specs:
if spec.kind != kind:
continue
merged.dev = merged.dev or spec.dev
merged.spec_path = merged.spec_path or spec.spec_path
merged.packages += spec.packages
return merged
def merge_install_specs(specs: list[InstallSpec]) -> InstallSpec:
"""Combine a language's install blocks into the environment they mean."""
merged = InstallSpec(kind=specs[0].kind)
seen: set[str] = set()
for spec in specs:
merged.dev = merged.dev or spec.dev
merged.spec_path = merged.spec_path or spec.spec_path
for package in spec.packages:
key = _requirement_key(package)
if key in seen:
continue
seen.add(key)
merged.packages.append(package)
return merged
# Where a package of each language keeps its own spec and its source, so
# a project can recognize an install of itself.
LOCAL_PACKAGE_SPECS: dict[str, tuple[str, list[str]]] = {
# Python's source directory is named for the module, so it is worked
# out from the package name rather than listed here
"python": ("pyproject.toml", []),
"julia": ("Project.toml", ["src"]),
"r": ("DESCRIPTION", ["R"]),
}
def local_package_name(language: str, wdir: str | None = None) -> str | None:
"""Return the name of the package the project itself defines, if any."""
entry = LOCAL_PACKAGE_SPECS.get(language)
if entry is None:
return None
fname = entry[0]
path = os.path.join(wdir, fname) if wdir else fname
if not os.path.isfile(path):
return None
try:
with open(path, encoding="utf-8") as f:
text = f.read()
except OSError:
return None
if fname == "DESCRIPTION":
m = re.search(r"^Package:\s*(\S+)\s*$", text, re.MULTILINE)
return m.group(1) if m else None
try:
import tomllib
data = tomllib.loads(text)
except Exception:
return None
if fname == "Project.toml":
name = data.get("name")
else:
name = (data.get("project") or {}).get("name")
return str(name) if name else None
def local_package_uuid(wdir: str | None = None) -> str | None:
"""Return the UUID of the Julia package the project itself defines."""
path = os.path.join(wdir, "Project.toml") if wdir else "Project.toml"
if not os.path.isfile(path):
return None
try:
import tomllib
with open(path, "rb") as f:
uuid = tomllib.load(f).get("uuid")
except Exception:
return None
return str(uuid) if uuid else None
def env_local_source_language(
env: dict, wdir: str | None = None
) -> str | None:
"""Return the language of an env that installs the project's own package.
That is either an environment pointing straight at the project's own
spec file, or a generated one that names the project root as a path
dependency. Either way the install is a pointer to the working tree,
so the package's source has to be a dependency of the stages using
the environment.
"""
language = ENV_KIND_LANGUAGES.get(str(env.get("kind", "")))
path = env.get("path")
if language is None or not path:
return None
spec_entry = LOCAL_PACKAGE_SPECS.get(language)
if spec_entry is not None and path == spec_entry[0]:
return language
full = os.path.join(wdir, path) if wdir else path
if not os.path.isfile(full):
return None
try:
with open(full, encoding="utf-8") as f:
content = f.read()
except OSError:
return None
return language if _rel_root_from_spec(path) in content else None
def local_package_source_paths(
language: str, wdir: str | None = None
) -> list[str]:
"""Return the paths holding the source of the project's own package.
A stage running in an environment that installed the project has to
rerun when the project's code changes, and the lock file says nothing
about that---an editable install is a pointer to the working tree.
"""
name = local_package_name(language, wdir=wdir)
entry = LOCAL_PACKAGE_SPECS.get(language)
if name is None or entry is None:
return []
_, source_dirs = entry
if language == "python":
# Distributions normalize the name; directories use the module
# spelling, which replaces separators with underscores
module = re.sub(r"[-.]+", "_", name)
candidates = [os.path.join("src", module), module]
else:
candidates = list(source_dirs)
found = []
for candidate in candidates:
full = os.path.join(wdir, candidate) if wdir else candidate
if os.path.isdir(full):
found.append(Path(candidate).as_posix())
return found
def installs_local_package(spec: InstallSpec, wdir: str | None = None) -> bool:
"""Determine whether an install names the project's own package.
A README tells a reader to install the package from wherever it is
published; read inside the package's own repo, that same line means
the copy in the working tree.
"""
name = local_package_name(spec.language, wdir=wdir)
if name is None:
return False
key = _requirement_key(name)
return any(_requirement_key(pkg) == key for pkg in spec.packages)
def resolve_dev_install(
spec: InstallSpec, wdir: str | None = None
) -> tuple[str, str] | None:
"""Find the spec file a dev install refers to, as (kind, path).
``pip install -e .`` says the environment is this project, which is
already described by whatever spec file sits in the project
directory---so the environment points at that file rather than one
Calkit generates.
"""
candidates = DEV_INSTALL_SPECS.get(spec.language, [])
# The installer that was named wins if its own spec file is there
ordered = sorted(candidates, key=lambda kp: kp[0] != spec.kind)
for kind, fname in ordered:
path = os.path.join(wdir, fname) if wdir else fname
if os.path.isfile(path):
return kind, fname
return None
LANGUAGE_DETECTORS: dict[str, str] = {
"python": "detect_python_dependencies",
"py": "detect_python_dependencies",
"r": "detect_r_dependencies",
"julia": "detect_julia_dependencies",
"jl": "detect_julia_dependencies",
}
# Suffix distinguishing one detected environment from another when a file
# needs more than one.
LANGUAGE_SUFFIXES: dict[str, str] = {
"python": "py",
"py": "py",
"r": "r",
"julia": "jl",
"jl": "jl",
}
# What to pin a Julia environment to when Julia isn't installed to ask.
FALLBACK_JULIA_VERSION = "1.12"
def detected_julia_version() -> str:
"""Return the installed Julia's major.minor version, or a fallback."""
import calkit.julia
try:
version = calkit.julia.get_version()
except Exception:
return FALLBACK_JULIA_VERSION
parts = version.split(".")
return ".".join(parts[:2]) if len(parts) >= 2 else version
def _env_base_name(markdown_path: str) -> str:
stem = Path(markdown_path).name.removesuffix(".md").lower()
cleaned = "".join(c if c.isalnum() else "-" for c in stem).strip("-")
return cleaned or "markdown"
def detect_environments(
specs: dict[str, MarkdownStageSpec],
markdown_path: str,
existing_env_names: list[str] | None = None,
default_env: str | None = None,
) -> tuple[dict[str, dict], dict[str, str]]:
"""Build environments for the stages in a file that declare none.
Stages are grouped by language and given one environment each, rather
than one apiece: a README's examples are normally variations on the
same setup, and a virtualenv per code block would be a lot of disk and
lock churn for nothing.
Returns the environments and a map of stage name -> environment name.
"""
import calkit.environments
from calkit import detect as _detect
existing = list(existing_env_names or [])
by_language: dict[str, list[MarkdownStageSpec]] = {}
for spec in specs.values():
if spec.attrs.get("environment") or default_env:
continue
if spec.language not in LANGUAGE_DETECTORS:
continue
by_language.setdefault(spec.language, []).append(spec)
if not by_language:
return {}, {}
base = _env_base_name(markdown_path)
envs: dict[str, dict] = {}
assignments: dict[str, str] = {}
for language, language_specs in sorted(by_language.items()):
detector = getattr(_detect, LANGUAGE_DETECTORS[language])
deps: list[str] = []
for spec in language_specs:
for dep in detector(code=spec.content):
if dep not in deps:
deps.append(dep)
# Suffix when this file needs more than one environment, and also
# when it has already named one that way, so a file's environments
# are named alike whether they were declared or detected
suffixed = len(by_language) > 1 or any(
n.startswith(f"{base}-") for n in existing
)
name = f"{base}-{LANGUAGE_SUFFIXES[language]}" if suffixed else base
candidate = name
n = 2
while candidate in existing or candidate in envs:
candidate = f"{name}-{n}"
n += 1
existing.append(candidate)
kind = LANGUAGE_ENV_KINDS[language]
env: dict[str, Any] = {
"kind": kind,
"path": get_env_spec_path(candidate, kind=kind),
"_spec_content": render_env_spec(kind, "\n".join(deps), candidate),
}
if kind == "uv":
# Pin the interpreter rather than letting it float above the
# requires-python floor, so the lock means something
env["_python"] = calkit.environments.DEFAULT_PYTHON_VERSION
elif kind == "julia":
# A Julia environment has no default version to fall back on,
# so it is pinned to whatever Julia is installed here
env["julia"] = detected_julia_version()
envs[candidate] = env
for spec in language_specs:
assignments[spec.name] = candidate
return envs, assignments
def format_attr_value(value: Any) -> str:
"""Render an attribute value the way an annotation spells it."""
if value is True:
return ""
if isinstance(value, str):
# Quote only when the value would otherwise not survive a
# round-trip through the attribute scanner
if value and not any(c.isspace() for c in value):
return value
return '"' + value.replace('"', '\\"') + '"'
if isinstance(value, (list, tuple)):
return "[" + ", ".join(format_attr_value(v) for v in value) + "]"
if isinstance(value, dict):
return (
"{"
+ ", ".join(
f"{k}: {format_attr_value(v)}" for k, v in value.items()
)
+ "}"
)
return str(value)
def format_attrs(attrs: dict[str, Any]) -> str:
"""Render attributes back into an annotation's key=value form."""
parts = []
for key, value in attrs.items():
rendered = format_attr_value(value)
parts.append(key if rendered == "" else f"{key}={rendered}")
return " ".join(parts)
# Languages a plain code fence can be turned into a stage automatically.
# Deliberately narrower than what Calkit can run: a README's shell blocks
# are usually instructions for the reader---install this, clone that---and
# quietly promoting them to pipeline stages would run them.
AUTO_STAGE_LANGUAGES = tuple(LANGUAGE_DETECTORS)
@dataclass
class Annotation:
"""What annotating a plain Markdown file added to it."""
# Stage name -> language, and environment name -> the language whose
# stages it serves
stages: dict[str, str] = field(default_factory=dict)
environments: dict[str, str] = field(default_factory=dict)
def __bool__(self) -> bool:
return bool(self.stages or self.environments)
@dataclass
class _Fence:
"""One code fence found while scanning a file to annotate it."""
index: int
indent: str
fence: str
info: str
ending: str
language: str | None
install: InstallSpec | None = None
transcript: bool = False
def _scan_fences(lines: list[str]) -> list[_Fence]:
"""Find the un-annotated code fences in a file, outermost first."""
fences: list[_Fence] = []
i = 0
while i < len(lines):
raw = lines[i].rstrip("\r\n")
m = _FENCE_RE.match(raw)
if m is None:
i += 1
continue
fence = m.group("fence")
info = m.group("info").strip()
# Find this fence's closing line first, so a longer fence's nested
# examples aren't mistaken for blocks of their own
j = i + 1
while j < len(lines):
close = _FENCE_RE.match(lines[j].rstrip("\r\n"))
if (
close is not None
and close.group("fence")[0] == fence[0]
and len(close.group("fence")) >= len(fence)
and not close.group("info").strip()
):
break
j += 1
try:
language, annotation = _parse_fence_info(info)
except ValueError:
language, annotation = None, None
if annotation is None and language is not None:
content = "".join(lines[i + 1 : j])
fences.append(
_Fence(
index=i,
indent=m.group("indent"),
fence=fence,
info=info,
ending=lines[i][len(raw) :],
language=language,
install=parse_install_block(content, language),
transcript=is_repl_transcript(content),
)
)
i = j + 1
return fences
def annotate_code_blocks(
text: str, markdown_path: str
) -> tuple[str, Annotation]:
"""Annotate a plain file's code fences so its blocks become a pipeline.
This is what makes ``calkit xr README.md`` work on a README nobody has
marked up yet. Code blocks are named for their language and so join
into one stage per language, in document order, because consecutive
blocks in a README read as one continuing script---what block two
imports, block one is expected to have set up.
Blocks that install packages are read as the environment those stages
run in, since showing the install command is how a README says what
its code needs.
Only info strings are touched, never prose and never code. Fences
that already carry a Calkit annotation are left alone.
"""
import calkit.environments
lines = text.splitlines(keepends=True)
fences = _scan_fences(lines)
stage_fences = [
f
for f in fences
if f.install is None
and not f.transcript
and f.language is not None
and f.language.lower() in AUTO_STAGE_LANGUAGES
]
install_fences = [f for f in fences if f.install is not None]
stage_names = {
LANGUAGE_SUFFIXES[f.language.lower()] # type: ignore[union-attr]
for f in stage_fences
}
# An environment nothing runs in is not worth building, so only the
# installs for a language the file actually has code in are read
env_suffixes = {
LANGUAGE_SUFFIXES[f.install.language] # type: ignore[union-attr]
for f in install_fences
} & stage_names
base = _env_base_name(markdown_path)
multi = len(stage_names) > 1
env_names = {
suffix: (f"{base}-{suffix}" if multi else base)
for suffix in sorted(env_suffixes)
}
result = Annotation()
for f in install_fences:
install = f.install
if install is None:
continue
suffix = LANGUAGE_SUFFIXES[install.language]
if suffix not in env_names:
continue
name = env_names[suffix]
annotation = f"calkit environment name={name}"
# Pin the interpreter here rather than letting it float, so what
# the environment resolves to doesn't depend on when it is built
if install.kind == "julia":
annotation += f" julia={detected_julia_version()}"
elif install.kind in ("uv", "uv-venv") and not install.dev:
annotation += (
f" python={calkit.environments.DEFAULT_PYTHON_VERSION}"
)
_annotate_fence(lines, f, annotation)
result.environments[name] = install.language
for f in stage_fences:
suffix = LANGUAGE_SUFFIXES[f.language.lower()] # type: ignore
annotation = f"calkit stage name={suffix}"
env_name = env_names.get(suffix)
if env_name is not None:
annotation += f" environment={env_name}"
_annotate_fence(lines, f, annotation)
result.stages[suffix] = f.language.lower() # type: ignore
return "".join(lines), result
def _annotate_fence(lines: list[str], fence: _Fence, annotation: str) -> None:
"""Append an annotation to a fence's info string, in place."""
# The original info string is kept whole rather than rebuilt from the
# language, so any other renderer's attributes on the fence survive
lines[fence.index] = (
fence.indent
+ fence.fence
+ fence.info
+ " "
+ annotation
+ fence.ending
)
def set_stage_attrs(
text: str, stage_name: str, attrs: dict[str, Any]
) -> tuple[str, bool]:
"""Add attributes to a stage's first annotated fence in Markdown text.
Only the info string of a block that is already annotated is touched,
never prose and never code, so this can record what was detected
without rewriting anything the author wrote. Attributes already set
are left alone, since the author's word beats a detected one.
Returns the new text and whether anything changed.
"""
if not attrs:
return text, False
lines = text.splitlines(keepends=True)
bare = [line.rstrip("\r\n") for line in lines]
i = 0
while i < len(lines):
line = lines[i]
m = _FENCE_RE.match(bare[i])
if m is not None:
try:
language, annotation = _parse_fence_info(m.group("info"))
except ValueError:
annotation = None
if (
annotation is not None
and annotation[0] == "stage"
and str(annotation[1].get("name")) == stage_name
):
new_attrs = {
k: v for k, v in attrs.items() if k not in annotation[1]
}
if not new_attrs:
return text, False
ending = line[len(bare[i]) :]
info = " ".join(
part
for part in [
language or "",
"calkit stage",
format_attrs(annotation[1]),
format_attrs(new_attrs),
]
if part
)
lines[i] = m.group("indent") + m.group("fence") + info + ending
return "".join(lines), True
i += 1
continue
# A stage declared in a directive comment gets the attributes
# written into the comment, just before it closes
try:
annotation, end = _read_comment_directive(bare, i)
except ValueError:
annotation, end = None, i + 1
if (
annotation is not None
and annotation[0] == "stage"
and str(annotation[1].get("name")) == stage_name
):
new_attrs = {
k: v for k, v in attrs.items() if k not in annotation[1]
}
if not new_attrs:
return text, False
last = end - 1
close_at = bare[last].rfind("-->")
before = bare[last][:close_at].rstrip()
ending = lines[last][len(bare[last]) :]
lines[last] = (
" ".join(
p for p in [before, format_attrs(new_attrs), "-->"] if p
)
+ ending
)
return "".join(lines), True
i = max(end, i + 1)
return text, False
def set_output_blocks(text: str, outputs: dict[str, str]) -> tuple[str, bool]:
"""Replace the bodies of ``calkit output`` blocks with what ran.
``outputs`` maps a stage name to its captured standard output. Only
the body of a block already annotated as an output for that stage is
replaced; a stage with no output block is simply not shown, and a
block whose stage produced nothing is emptied rather than left stale.
Returns the new text and whether anything changed.
"""
if not outputs:
return text, False
lines = text.splitlines(keepends=True)
out_lines: list[str] = []
changed = False
i = 0
while i < len(lines):
line = lines[i]
m = _FENCE_RE.match(line.rstrip("\r\n"))
if m is None:
out_lines.append(line)
i += 1
continue
fence = m.group("fence")
try:
_, annotation = _parse_fence_info(m.group("info"))
except ValueError:
annotation = None
# Find this fence's closing line either way, so a block we don't
# touch is copied through without its contents being rescanned
j = i + 1
while j < len(lines):
close = _FENCE_RE.match(lines[j].rstrip("\r\n"))
if (
close is not None
and close.group("fence")[0] == fence[0]
and len(close.group("fence")) >= len(fence)
and not close.group("info").strip()
):
break
j += 1
stage_name = (
str(annotation[1].get("name") or annotation[1].get("stage"))
if annotation is not None and annotation[0] == "output"
else None
)
if stage_name is None or stage_name not in outputs:
out_lines += lines[i : j + 1]
i = j + 1
continue
body = outputs[stage_name]
new_body = [line + "\n" for line in body.splitlines()] if body else []
# Output can itself contain a line that would close the block, so
# the fence is lengthened past the longest such run in it; a fence
# that is already long enough is left as it is
longest = max(
(
len(fm.group("fence"))
for fm in (_FENCE_RE.match(b.rstrip("\r\n")) for b in new_body)
if fm is not None and fm.group("fence")[0] == fence[0]
),
default=0,
)
open_line = line
close_line = lines[j] if j < len(lines) else None
if longest >= len(fence):
new_fence = fence[0] * (longest + 1)
open_line = (
m.group("indent")
+ new_fence
+ m.group("info")
+ line[len(line.rstrip("\r\n")) :]
)
if close_line is not None:
cm = _FENCE_RE.match(close_line.rstrip("\r\n"))
assert cm is not None
close_line = (
cm.group("indent")
+ new_fence
+ close_line[len(close_line.rstrip("\r\n")) :]
)
if lines[i + 1 : j] != new_body or open_line != line:
changed = True
out_lines.append(open_line)
out_lines += new_body
if close_line is not None:
out_lines.append(close_line)
i = j + 1
return "".join(out_lines), changed
# A value marker wraps the text it keeps current, so the file reads as
# ordinary prose wherever it is rendered and the number is still there to
# be checked. Both comments sit on one line with the value between them.
_VALUE_RE = re.compile(
r"<!--\s*calkit value\b(?P<attrs>.*?)-->"
r"(?P<text>.*?)"
r"<!--\s*/calkit value\s*-->"
)
_VALUES_RE = re.compile(r"<!--\s*calkit values\b(?P<attrs>.*?)-->")
@dataclass
class MarkdownValue:
"""One value a Markdown file shows from a JSON results file."""
key: str
path: str
# What the file currently shows for it
text: str
format: str | None = None
# 1-indexed line the marker is on
line: int = 0
def _value_attrs(attrs_text: str, path: str | None, line: int) -> dict:
try:
return parse_attrs(attrs_text)
except ValueError as e:
raise MarkdownParseError(str(e), path=path, line=line)
def extract_values(text: str, markdown_path: str) -> list[MarkdownValue]:
"""Find the value markers in Markdown text, in document order.
A ``calkit values path=...`` directive sets the results file for the
markers after it, so a file drawing on one results file needn't name
it on every value. Markers inside fenced code are examples, not
values, and are left alone.
"""
path = Path(markdown_path).as_posix()
values: list[MarkdownValue] = []
default_path: str | None = None
fence: str | None = None
for i, line in enumerate(text.splitlines()):
fm = _FENCE_RE.match(line)
if fence is not None:
if (
fm is not None
and fm.group("fence")[0] == fence[0]
and len(fm.group("fence")) >= len(fence)
and not fm.group("info").strip()
):
fence = None
continue
if fm is not None:
fence = fm.group("fence")
continue
for dm in _VALUES_RE.finditer(line):
attrs = _value_attrs(dm.group("attrs"), path, i + 1)
if set(attrs) != {"path"}:
raise MarkdownParseError(
"'calkit values' takes only a 'path', e.g. "
"'calkit values path=results.json'",
path=path,
line=i + 1,
)
default_path = Path(str(attrs["path"])).as_posix()
for m in _VALUE_RE.finditer(line):
attrs = _value_attrs(m.group("attrs"), path, i + 1)
key = attrs.pop("key", None)
if key is None:
raise MarkdownParseError(
"A value marker must name a key, e.g. "
"'calkit value key=rms'",
path=path,
line=i + 1,
)
results_path = attrs.pop("path", default_path)
if results_path is None:
raise MarkdownParseError(
f"Value '{key}' names no results file; set 'path=' on "
"the marker or declare one with 'calkit values "
"path=...' above it",
path=path,
line=i + 1,
)
fmt = attrs.pop("format", None)
if attrs:
raise MarkdownParseError(
f"Value '{key}' has unrecognized attribute(s): "
f"{', '.join(sorted(attrs))}",
path=path,
line=i + 1,
)
values.append(
MarkdownValue(
key=str(key),
path=Path(str(results_path)).as_posix(),
text=m.group("text"),
format=None if fmt is None else str(fmt),
line=i + 1,
)
)
return values
def lookup_value(data: Any, key: str) -> Any:
"""Return the value at a dotted key, e.g. ``fit.coeffs.0``."""
value = data
for part in key.split("."):
if isinstance(value, dict) and part in value:
value = value[part]
elif isinstance(value, list) and part.lstrip("-").isdigit():
value = value[int(part)]
else:
raise KeyError(key)
return value
def format_value(value: Any, fmt: str | None = None) -> str:
"""Render a value the way it should read in prose.
``fmt`` is a Python format string applied to the value, e.g.
``{:.3f}`` or ``{value:,}``. Without one, a number is written as
JSON would write it, so the file shows exactly what the results file
holds.
"""
import json
if fmt is not None:
return fmt.format(value, value=value)
if isinstance(value, str):
return value
return json.dumps(value)
def set_values(
text: str, markdown_path: str, wdir: str | None = None
) -> tuple[str, bool]:
"""Rewrite each value marker's text from its results file.
The results files are outputs of stages, so they are the truth about
what was computed; the prose is brought into line with them rather
than the other way round. Returns the new text and whether anything
changed. A missing file or key raises ``MarkdownParseError`` naming
the line.
"""
import json
values = extract_values(text, markdown_path)
if not values:
return text, False
path = Path(markdown_path).as_posix()
data_by_path: dict[str, Any] = {}
for value in values:
if value.path in data_by_path:
continue
fpath = os.path.join(wdir, value.path) if wdir else value.path
if not os.path.isfile(fpath):
raise MarkdownParseError(
f"Results file '{value.path}' for value '{value.key}' does "
"not exist; is it an output of a stage that has run?",
path=path,
line=value.line,
)
with open(fpath, encoding="utf-8") as f:
try:
data_by_path[value.path] = json.load(f)
except json.JSONDecodeError as e:
raise MarkdownParseError(
f"Results file '{value.path}' is not valid JSON: {e}",
path=path,
line=value.line,
)
rendered = []
for value in values:
try:
raw = lookup_value(data_by_path[value.path], value.key)
except KeyError:
raise MarkdownParseError(
f"Results file '{value.path}' has no key '{value.key}'",
path=path,
line=value.line,
)
try:
rendered.append(format_value(raw, value.format))
except Exception as e:
raise MarkdownParseError(
f"Could not format value '{value.key}' with "
f"'{value.format}': {e}",
path=path,
line=value.line,
)
# Markers were found outside fences in document order, so walking the
# lines again the same way pairs each match with its rendering
lines = text.splitlines(keepends=True)
changed = False
n = 0
fence: str | None = None
for i, line in enumerate(lines):
bare = line.rstrip("\r\n")
fm = _FENCE_RE.match(bare)
if fence is not None:
if (
fm is not None
and fm.group("fence")[0] == fence[0]
and len(fm.group("fence")) >= len(fence)
and not fm.group("info").strip()
):
fence = None
continue
if fm is not None:
fence = fm.group("fence")
continue
matches = list(_VALUE_RE.finditer(bare))
if not matches:
continue
# Replace from the right so earlier offsets stay valid
for m in reversed(matches):
new = rendered[n + matches.index(m)]
if new != m.group("text"):
bare = bare[: m.start("text")] + new + bare[m.end("text") :]
changed = True
n += len(matches)
lines[i] = bare + line[len(line.rstrip("\r\n")) :]
return "".join(lines), changed
"""Functionality for working with runnable Markdown files.
A Markdown file can declare pipeline stages and environments by annotating
its code blocks, so a README can be the source of truth for what it
documents. An annotated block looks like::
```python calkit stage name=example environment=main
print("hello")
```
The first token of a fence's info string stays the language, since
Markdown renderers use it for syntax highlighting and ignore the rest.
Annotations can also be written as an HTML comment directive immediately
preceding a block, which keeps long attribute lists out of the info string
and lets a plain bulleted list declare an environment's dependencies::
<!-- calkit environment name=main python=3.13 -->
- numpy
- matplotlib
"""
from __future__ import annotations
import os
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
import ruamel.yaml
from pydantic import BaseModel
DirectiveKind = Literal["stage", "environment", "output", "value", "values"]
# ``output`` blocks are written to, never read: a stage's extracted script
# must not include what that stage printed, or injecting output would make
# the stage stale, which would rerun it, which would inject again.
# ``value`` and ``values`` are inline markers rather than block
# annotations; see ``set_values``.
DIRECTIVE_KINDS: tuple[str, ...] = (
"stage",
"environment",
"output",
"value",
"values",
)
BlockSource = Literal["fence", "list"]
# A fence is at least three backticks or tildes, optionally indented. Only
# the run of fence characters and the info string that follows are captured;
# the run's length matters because a longer fence can contain shorter ones,
# which is how a Markdown file documents this very feature without
# declaring the examples it shows.
_FENCE_RE = re.compile(
r"^(?P<indent>[ ]{0,3})(?P<fence>`{3,}|~{3,})(?P<info>.*)$"
)
_LIST_ITEM_RE = re.compile(r"^[ ]{0,3}[-*+][ ]+(?P<item>.*?)\s*$")
_COMMENT_OPEN_RE = re.compile(r"^[ ]{0,3}<!--\s*(?P<body>.*)$")
_yaml = ruamel.yaml.YAML(typ="safe")
class MarkdownParseError(ValueError):
"""Raised when a Calkit annotation in a Markdown file can't be read."""
def __init__(self, message: str, path: str | None = None, line: int = 0):
self.path = path
self.line = line
where = f"{path}:{line}" if path else f"line {line}"
super().__init__(f"{where}: {message}")
# Names are used as path components---a stage's extracted script and an
# environment's spec directory are named for them---so they must be a
# single safe segment rather than something that escapes the project.
_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
def check_name(name: str, what: str, path: str | None, line: int) -> None:
"""Check that a declared name can be used as a single path segment."""
if not _NAME_RE.match(name) or name in (".", ".."):
raise MarkdownParseError(
f"{what} name '{name}' is not valid; names may only contain "
"letters, digits, '.', '_' and '-', and must start with a "
"letter or digit",
path=path,
line=line,
)
class MarkdownBlock:
"""A Calkit-annotated block extracted from a Markdown file."""
kind: DirectiveKind
attrs: dict[str, Any] = field(default_factory=dict)
content: str = ""
language: str | None = None
source: BlockSource = "fence"
# 1-indexed line at which the block's annotation starts, for error
# messages that point back at what the author wrote.
Xet Storage Details
- Size:
- 101 kB
- Xet hash:
- 47ed312c0ffbb1a22c60de5f2f43e78baf8d6a846848472e5d819b9dd37db775
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.