Buckets:
"""Functionality for detecting inputs, outputs, and dependencies from scripts,
notebooks, and commands.
"""
from __future__ import annotations
import ast
import json
import os
import re
import shlex
import sys
from collections.abc import Iterable
from pathlib import Path
from typing import Literal
import pathspec
NotebookLanguage = Literal["python", "julia", "r"]
def detect_python_script_io(
script_path: str,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from a Python script using static analysis."""
inputs = []
outputs = []
if not os.path.exists(script_path):
return {"inputs": inputs, "outputs": outputs}
try:
with open(script_path, "r", encoding="utf-8") as f:
content = f.read()
except UnicodeDecodeError:
return {"inputs": inputs, "outputs": outputs}
script_dir = os.path.dirname(script_path) if script_path else "."
effective_wdir = wdir or "."
return _detect_python_code_io(
content,
script_dir=script_dir,
working_dir=effective_wdir,
current_dir=effective_wdir,
)
def detect_r_script_io(
script_path: str,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from an R script using regex patterns.
Parameters
----------
script_path : str
Path to the R script file.
wdir : str | None
Working directory from which the script will be executed.
Defaults to current directory.
Returns
-------
dict[str, list[str]]
Dictionary with 'inputs' and 'outputs' keys.
"""
inputs = []
outputs = []
if not os.path.exists(script_path):
return {"inputs": inputs, "outputs": outputs}
try:
with open(script_path, "r", encoding="utf-8") as f:
content = f.read()
except UnicodeDecodeError:
return {"inputs": inputs, "outputs": outputs}
script_dir = os.path.dirname(script_path) if script_path else "."
effective_wdir = wdir or "."
return _detect_r_code_io(
content,
script_dir=script_dir,
wdir=effective_wdir,
project_root=effective_wdir,
)
def detect_julia_script_io(
script_path: str,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from a Julia script using regex patterns.
Parameters
----------
script_path : str
Path to the Julia script file.
wdir : str | None
Working directory from which the script will be executed.
Defaults to current directory.
Returns
-------
dict[str, list[str]]
Dictionary with 'inputs' and 'outputs' keys.
"""
inputs = []
outputs = []
if not os.path.exists(script_path):
return {"inputs": inputs, "outputs": outputs}
try:
with open(script_path, "r", encoding="utf-8") as f:
content = f.read()
except UnicodeDecodeError:
return {"inputs": inputs, "outputs": outputs}
script_dir = os.path.dirname(script_path) if script_path else "."
effective_wdir = wdir or "."
content = re.sub(r"#.*$", "", content, flags=re.MULTILINE)
julia_vars = _extract_julia_string_assignments(content)
include_pattern = r'include\s*\(\s*["\']([^"\']+\.jl)["\']\s*\)'
include_matches = re.findall(include_pattern, content)
for match in include_matches:
if not os.path.isabs(match):
full_path = os.path.join(script_dir, match)
if os.path.exists(full_path):
inputs.append(Path(os.path.relpath(full_path)).as_posix())
elif _is_valid_project_path(match):
inputs.append(match)
read_patterns = [
r'open\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))\s*,\s*["\']r["\']',
r'open\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))\s*\)',
r'CSV\.(?:read|File)\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'readdlm\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'load\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'JLD2?\.load\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
]
write_patterns = [
r'open\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))\s*,\s*["\']w["\']',
r'open\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))\s*,\s*["\']a["\']',
r'CSV\.write\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'writedlm\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'save\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'JLD2?\.save\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'savefig\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
]
for pattern in read_patterns:
matches = re.findall(pattern, content)
for match in matches:
resolved = _resolve_julia_path_expr(match, julia_vars)
if resolved:
inputs.append(resolved)
for pattern in write_patterns:
matches = re.findall(pattern, content)
for match in matches:
resolved = _resolve_julia_path_expr(match, julia_vars)
if resolved:
outputs.append(resolved)
inputs = [p for p in inputs if _is_valid_project_path(p)]
outputs = [p for p in outputs if _is_valid_project_path(p)]
# Resolve paths relative to working directory
inputs = _resolve_paths_to_wdir(inputs, script_dir, effective_wdir)
outputs = _resolve_paths_to_wdir(outputs, script_dir, effective_wdir)
inputs = list(dict.fromkeys(inputs))
outputs = list(dict.fromkeys(outputs))
return {"inputs": inputs, "outputs": outputs}
def detect_shell_script_io(
script_path: str,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from a shell script using regex patterns.
Parameters
----------
script_path : str
Path to the shell script file.
wdir : str | None
Working directory from which the script will be executed.
Defaults to current directory.
Returns
-------
dict[str, list[str]]
Dictionary with 'inputs' and 'outputs' keys.
"""
inputs = []
outputs = []
if not os.path.exists(script_path):
return {"inputs": inputs, "outputs": outputs}
try:
with open(script_path, "r", encoding="utf-8") as f:
content = f.read()
except UnicodeDecodeError:
return {"inputs": inputs, "outputs": outputs}
script_dir = os.path.dirname(script_path) if script_path else "."
effective_wdir = wdir or "."
content = re.sub(r"#.*$", "", content, flags=re.MULTILINE)
# Output patterns (>> must come before > to avoid matching
# the first > in >>)
output_patterns = [
r">>\s*(\S+)", # append redirection
r"(?<!>)>\s*(\S+)", # single redirection, not part of '>>'
r"\bcp\s+\S+\s+(\S+)",
r"\bmv\s+\S+\s+(\S+)",
]
for pattern in output_patterns:
matches = re.findall(pattern, content)
for match in matches:
match = match.strip("\"'")
outputs.append(match)
# For input patterns, look for files with extensions after commands
input_patterns = [
r"<\s+(\S+)",
r"\bcat\s+([\S\.]+)",
r"\bhead\s+.*?([\S\.]+\.[\w]+)",
r"\btail\s+.*?([\S\.]+\.[\w]+)",
r"\bgrep\s+[^\s]+\s+([\S\.]+)",
r"\bawk\s+[^\s]+\s+([\S\.]+)",
r"\bsed\s+[^\s]+\s+([\S\.]+)",
r"\bcut\s+[^\s]+\s+([\S\.]+)",
r"\bsort\s+([\S\.]+)",
]
for pattern in input_patterns:
matches = re.findall(pattern, content)
for match in matches:
# Handle multiple files
for file in match.split():
file = file.strip("\"'").strip()
if (
file
and not file.startswith("-")
and not file.startswith("/dev/")
):
inputs.append(file)
inputs = [p for p in inputs if _is_valid_project_path(p)]
outputs = [p for p in outputs if _is_valid_project_path(p)]
# Resolve paths relative to working directory
inputs = _resolve_paths_to_wdir(inputs, script_dir, effective_wdir)
outputs = _resolve_paths_to_wdir(outputs, script_dir, effective_wdir)
inputs = list(dict.fromkeys(inputs))
outputs = list(dict.fromkeys(outputs))
return {"inputs": inputs, "outputs": outputs}
def _resolve_paths_to_wdir(
paths: list[str], script_dir: str, wdir: str
) -> list[str]:
"""Resolve paths from script directory to working directory.
When a script is in a subdirectory (e.g., scripts/process.R),
its relative paths should be resolved from the working directory
where the command is executed, not from the script's directory.
Parameters
----------
paths : list[str]
List of file paths detected in the script.
script_dir : str
Directory containing the script.
wdir : str
Working directory from which the script will be executed.
Returns
-------
list[str]
Paths resolved relative to wdir.
"""
if script_dir == wdir or script_dir == ".":
return paths
resolved = []
for path in paths:
if os.path.isabs(path):
# Absolute paths stay as-is (though they should be filtered already)
resolved.append(path)
else:
# Path is relative - assume it's relative to wdir, not script_dir
# Just pass it through as-is
resolved.append(path)
return resolved
def _extract_directory_changes(code: str, current_dir: str = ".") -> str:
"""Extract final working directory after all %cd and os.chdir() calls in
code.
Parameters
----------
code : str
Python code potentially containing %cd magic or os.chdir() calls.
current_dir : str
Starting directory for resolution.
Returns
-------
str
Final working directory after all changes.
"""
working_dir = current_dir
# Handle Jupyter %cd magic commands
for line in code.split("\n"):
stripped = line.strip()
if stripped.startswith("%cd "):
chdir_path = stripped[4:].strip()
chdir_path = chdir_path.strip("'\"")
if chdir_path:
working_dir = os.path.normpath(
os.path.join(working_dir, chdir_path)
)
# Handle os.chdir() calls
try:
tree = ast.parse(code)
for node in ast.walk(tree):
if isinstance(node, ast.Call):
if isinstance(node.func, ast.Attribute):
if (
isinstance(node.func.value, ast.Name)
and node.func.value.id == "os"
and node.func.attr == "chdir"
and len(node.args) >= 1
):
chdir_path = _extract_string_from_node(node.args[0])
if chdir_path:
working_dir = os.path.normpath(
os.path.join(working_dir, chdir_path)
)
except SyntaxError:
pass
return working_dir
def language_from_notebook(
notebook_path: str, notebook: dict | None = None
) -> NotebookLanguage | None:
"""Detect notebook language from kernelspec metadata."""
if notebook is None:
if not os.path.exists(notebook_path):
return
try:
with open(notebook_path, "r", encoding="utf-8") as f:
notebook = json.load(f)
except (json.JSONDecodeError, UnicodeDecodeError):
return
assert notebook is not None
metadata = notebook.get("metadata", {})
kernel_info = metadata.get("kernelspec", {})
kernel_lang = kernel_info.get("language", "").lower()
if "python" in kernel_lang:
return "python"
if "julia" in kernel_lang:
return "julia"
if kernel_lang == "r":
return "r"
return
def detect_jupyter_notebook_io(
notebook_path: str,
language: NotebookLanguage | None = None,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from a Jupyter notebook.
Extracts all code cells and passes them to the appropriate
language-specific detection function. Detection functions handle
variable tracking and directory changes across cells.
"""
inputs = []
outputs = []
if not os.path.exists(notebook_path):
return {"inputs": inputs, "outputs": outputs}
try:
with open(notebook_path, "r", encoding="utf-8") as f:
nb = json.load(f)
except (json.JSONDecodeError, UnicodeDecodeError):
return {"inputs": inputs, "outputs": outputs}
if language is None:
detected = language_from_notebook(notebook_path, notebook=nb)
language = detected if detected is not None else "python"
code_cells = [
cell for cell in nb.get("cells", []) if cell.get("cell_type") == "code"
]
# Concatenate all code cells with newlines to preserve structure
full_code = "\n".join(
"".join(cell.get("source", []))
if isinstance(cell.get("source", []), list)
else cell.get("source", "")
for cell in code_cells
)
notebook_dir = os.path.dirname(notebook_path) if notebook_path else "."
effective_wdir = wdir or "."
if language == "python":
return _detect_python_code_io(
full_code,
script_dir=notebook_dir,
working_dir=effective_wdir,
current_dir=notebook_dir,
)
elif language == "julia":
return _detect_julia_code_io(full_code, script_dir=notebook_dir)
elif language == "r":
return _detect_r_code_io(full_code, script_dir=notebook_dir)
else:
return {"inputs": [], "outputs": []}
def detect_latex_io(tex_path: str) -> dict[str, list[str]]:
"""Detect inputs and outputs from a LaTeX file using static analysis."""
inputs = []
outputs = []
if not os.path.exists(tex_path):
return {"inputs": inputs, "outputs": outputs}
# Get the directory of the LaTeX file for resolving relative references
tex_dir = os.path.dirname(tex_path)
try:
with open(tex_path, "r", encoding="utf-8") as f:
content = f.read()
except UnicodeDecodeError:
return {"inputs": inputs, "outputs": outputs}
# Remove comments
content = re.sub(r"(?<!\\)%.*$", "", content, flags=re.MULTILINE)
# Define patterns with their handling types
patterns = [
(r"\\input\{([^}]+)\}", "tex"),
(r"\\include\{([^}]+)\}", "tex"),
(r"\\includegraphics(?:\[[^\]]*\])?\{([^}]+)\}", "file"),
(r"\\bibliography\{([^}]+)\}", "bib"),
(r"\\addbibresource\{([^}]+)\}", "bib"),
]
for pattern, pattern_type in patterns:
matches = re.findall(pattern, content)
for match in matches:
def _resolve(name: str) -> str:
return Path(
os.path.normpath(os.path.join(tex_dir, name))
).as_posix()
if pattern_type == "bib":
files = [f.strip() for f in match.split(",")]
for f in files:
if not f.endswith(".bib"):
f += ".bib"
# Resolve relative to the document directory
inputs.append(_resolve(f))
elif pattern_type == "tex":
filename = match
if not filename.endswith(".tex"):
filename += ".tex"
# Resolve relative to the document directory
inputs.append(_resolve(filename))
else:
# Include graphics or other files
# Resolve relative to the document directory
inputs.append(_resolve(match))
# Filter and deduplicate inputs
inputs = [p for p in inputs if _is_valid_project_path(p)]
# The patterns above miss everything LaTeX resolves by name rather than
# by path: the document class, local style files, the bibliography
# style, and figures written without an extension. Those come from
# calkit.latex, which resolves against the project and follows the
# files it finds, so a class that loads its own styles contributes
# them too.
import calkit.latex
inputs += calkit.latex.detect_inputs(tex_path)
inputs = list(dict.fromkeys(inputs))
return {"inputs": inputs, "outputs": outputs}
def detect_shell_command_io(command: str) -> dict[str, list[str]]:
"""Detect inputs and outputs from a shell command string."""
from calkit.docker import extract_docker_run_inner_command
inputs = []
outputs = []
analyzed_command = command
inner_command = extract_docker_run_inner_command(command)
if inner_command is not None:
analyzed_command = " ".join(
shlex.quote(part) for part in inner_command
)
input_redirects = re.findall(r"<\s*(\S+)", analyzed_command)
inputs.extend([f.strip("\"'") for f in input_redirects])
output_redirects = re.findall(r">>?\s*(\S+)", analyzed_command)
outputs.extend([f.strip("\"'") for f in output_redirects])
try:
words = shlex.split(analyzed_command)
except ValueError:
words = analyzed_command.split()
if words and words[0] in ["cp", "mv"]:
operands = []
for word in words[1:]:
if word == "--":
continue
if word.startswith("-"):
continue
operands.append(word.strip("\"'"))
if len(operands) >= 2:
inputs.extend(operands[:-1])
outputs.append(operands[-1])
for i, word in enumerate(words):
if word.startswith("-"):
continue
if "/" in word or "." in word:
clean = word.strip("\"'")
if _is_valid_project_path(clean):
if i > 0 and words[0] in [
"cat",
"head",
"tail",
"grep",
"awk",
"sed",
]:
inputs.append(clean)
inputs = [p for p in inputs if _is_valid_project_path(p)]
outputs = [p for p in outputs if _is_valid_project_path(p)]
inputs = list(dict.fromkeys(inputs))
outputs = list(dict.fromkeys(outputs))
return {"inputs": inputs, "outputs": outputs}
def _extract_string_from_node(node: ast.AST) -> str | None:
"""Extract a string value from an AST node if it's a constant string."""
if isinstance(node, ast.Constant) and isinstance(node.value, str):
return node.value
return None
def _is_os_path_join(node: ast.Call) -> bool:
if isinstance(node.func, ast.Attribute):
if node.func.attr == "join":
value = node.func.value
if isinstance(value, ast.Attribute):
return (
value.attr == "path"
and isinstance(value.value, ast.Name)
and value.value.id == "os"
)
return False
def _resolve_join_call(
node: ast.Call, variables: dict[str, str]
) -> str | None:
parts: list[str] = []
for arg in node.args:
part = _resolve_path_expr(arg, variables)
if part is None:
return None
parts.append(part)
if not parts:
return "."
return os.path.join(*parts)
def _resolve_path_expr(node: ast.AST, variables: dict[str, str]) -> str | None:
# Direct string literal
path = _extract_string_from_node(node)
if path:
return path
# Variable reference
if isinstance(node, ast.Name) and node.id in variables:
return variables[node.id]
# Path composition with / or +
if isinstance(node, ast.BinOp):
if isinstance(node.op, (ast.Div, ast.Add)):
left = _resolve_path_expr(node.left, variables)
right = _resolve_path_expr(node.right, variables)
if left and right:
return os.path.join(left, right)
# Path attribute access
if isinstance(node, ast.Attribute):
if node.attr == "parent":
base = _resolve_path_expr(node.value, variables)
if base:
if base in [".", ""]:
return ".."
return os.path.dirname(base)
# Call expressions
if isinstance(node, ast.Call):
if _is_os_path_join(node):
return _resolve_join_call(node, variables)
if isinstance(node.func, ast.Name) and node.func.id == "join":
return _resolve_join_call(node, variables)
if isinstance(node.func, ast.Name) and node.func.id == "Path":
if not node.args:
return "."
return _resolve_join_call(node, variables)
if isinstance(node.func, ast.Attribute):
if node.func.attr == "joinpath":
base = _resolve_path_expr(node.func.value, variables)
if base:
parts = [base]
for arg in node.args:
part = _resolve_path_expr(arg, variables)
if part is None:
return None
parts.append(part)
return os.path.join(*parts)
if node.func.attr in ["absolute", "resolve"]:
base = _resolve_path_expr(node.func.value, variables)
if base:
return os.path.abspath(base)
if node.func.attr == "cwd":
if (
isinstance(node.func.value, ast.Name)
and node.func.value.id == "Path"
):
return "."
return None
def _extract_variable_assignments(tree: ast.Module) -> dict[str, str]:
"""Extract simple string variable assignments from AST.
Returns a dict mapping variable names to their string values.
Only tracks direct assignments like: var = "path/to/file"
"""
variables = {}
def visit_statements(statements: list[ast.stmt]) -> None:
for stmt in statements:
if isinstance(stmt, ast.Assign):
# Only handle simple single target assignments
if len(stmt.targets) == 1:
target = stmt.targets[0]
if isinstance(target, ast.Name):
value = _resolve_path_expr(stmt.value, variables)
if value:
variables[target.id] = value
elif isinstance(stmt, ast.AnnAssign):
if (
isinstance(stmt.target, ast.Name)
and stmt.value is not None
):
value = _resolve_path_expr(stmt.value, variables)
if value:
variables[stmt.target.id] = value
if isinstance(stmt, ast.If):
visit_statements(stmt.body)
visit_statements(stmt.orelse)
elif isinstance(stmt, (ast.For, ast.While)):
visit_statements(stmt.body)
visit_statements(stmt.orelse)
elif isinstance(stmt, (ast.With, ast.AsyncWith)):
visit_statements(stmt.body)
elif isinstance(stmt, ast.Try):
visit_statements(stmt.body)
visit_statements(stmt.orelse)
visit_statements(stmt.finalbody)
for handler in stmt.handlers:
visit_statements(handler.body)
if isinstance(tree, ast.Module):
visit_statements(tree.body)
return variables
def _resolve_variable_in_call(
node: ast.expr, variables: dict[str, str]
) -> str | None:
"""Try to resolve a file path from a call node that may reference a
variable.
Handles patterns like:
- open(variable)
- open("literal_path")
- path_template.format(...)
"""
# Direct path expression
path = _resolve_path_expr(node, variables)
if path:
return path
# For format calls: path_template.format(key=value) -> substitute template
if isinstance(node, ast.Call):
if isinstance(node.func, ast.Attribute):
if node.func.attr == "format":
# Get the string being formatted
base_path = _resolve_variable_in_call(
node.func.value, variables
)
if base_path:
# Extract keyword arguments and substitute them
format_kwargs = {}
for keyword in node.keywords:
if keyword.arg:
value = _resolve_path_expr(
keyword.value, variables
)
if value:
format_kwargs[keyword.arg] = value
if format_kwargs:
try:
return base_path.format(**format_kwargs)
except (KeyError, ValueError):
# If format fails, return the template
return base_path
return base_path
return None
def filter_covered_inputs(
detected: Iterable[str], existing: Iterable[str]
) -> list[str]:
"""Drop detected paths an existing input already covers.
A stage input can be a whole directory: with ``figures`` declared,
everything inside it is already a dependency, so listing those files
one by one adds noise without adding coverage -- and goes stale the
moment one is renamed. An exact repeat of an existing input is covered
by the same rule.
Order is preserved, and duplicates within ``detected`` are dropped.
"""
covered = {
path.rstrip("/") for path in existing if isinstance(path, str) and path
}
out: list[str] = []
for path in detected:
if not isinstance(path, str) or not path:
continue
parts = path.rstrip("/").split("/")
ancestors = ["/".join(parts[:i]) for i in range(1, len(parts) + 1)]
if any(ancestor in covered for ancestor in ancestors):
continue
out.append(path)
covered.add(path.rstrip("/"))
return out
def _is_valid_project_path(path: str) -> bool:
"""Check if a path is valid for a project dependency/output."""
if not path:
return False
if path.startswith(("http://", "https://", "ftp://", "s3://", "gs://")):
return False
if os.path.isabs(path):
return False
if path.startswith(("/dev/", "/proc/", "/sys/", "~")):
return False
return True
def _detect_python_code_io(
code: str,
script_dir: str = ".",
working_dir: str = ".",
current_dir: str | None = None,
) -> dict[str, list[str]]:
"""Detect I/O from Python code string (used for notebook cells
and scripts).
Tracks os.chdir() calls and Jupyter %cd magic commands to maintain the
effective working directory for resolving relative file paths. Supports
variable tracking for paths defined in variables. All paths are normalized
relative to the project root (current working directory).
When analyzing notebook code, all cells should be concatenated
into a single code string before calling this function. This
ensures variable assignments defined in earlier cells are
available for resolution in later cells.
"""
inputs = []
outputs = []
current_dir = current_dir or working_dir
import_dir = script_dir or "."
# First pass: detect directory changes from %cd magic commands
for line in code.split("\n"):
stripped = line.strip()
# Handle Jupyter %cd magic
if stripped.startswith("%cd "):
chdir_path = stripped[4:].strip()
# Remove quotes if present
chdir_path = chdir_path.strip("'\"")
if chdir_path:
current_dir = os.path.normpath(
os.path.join(current_dir, chdir_path)
)
# Remove Jupyter magic commands from code before parsing
# (they are not valid Python syntax)
code_for_parsing = "\n".join(
line if not line.strip().startswith("%") else ""
for line in code.split("\n")
)
try:
tree = ast.parse(code_for_parsing)
except SyntaxError:
# If the code cannot be parsed, we cannot reliably detect directory
# changes; keep the current working_dir unchanged.
return {"inputs": inputs, "outputs": outputs}
# Extract variable assignments for path tracking
variables = _extract_variable_assignments(tree)
# Second pass: identify all os.chdir() calls to track directory changes
for node in ast.walk(tree):
if isinstance(node, ast.Call):
if isinstance(node.func, ast.Attribute):
if (
isinstance(node.func.value, ast.Name)
and node.func.value.id == "os"
and node.func.attr == "chdir"
and len(node.args) >= 1
):
chdir_path = _resolve_variable_in_call(
node.args[0], variables
)
if chdir_path:
current_dir = os.path.normpath(
os.path.join(current_dir, chdir_path)
)
# Helper to resolve a relative path back to project root
def resolve_to_root(path: str) -> str:
"""Resolve a path from current_dir to project root."""
full_path = os.path.normpath(os.path.join(current_dir, path))
# Handle paths that reference parent directories
# by computing the real path
abs_path = os.path.abspath(full_path)
project_root = os.path.abspath(working_dir)
try:
rel_path = os.path.relpath(abs_path, project_root)
except ValueError:
# If paths are on different drives (Windows), use normalized path
rel_path = full_path
# Always emit posix separators so downstream comparisons are stable
# across platforms.
rel_path = Path(rel_path).as_posix()
# Strip leading ../ for paths outside project root
while rel_path.startswith("../"):
rel_path = rel_path[3:]
return rel_path
# Output methods for pandas-like objects
output_methods = {
"to_csv",
"to_excel",
"to_parquet",
"to_feather",
"to_pickle",
"to_json",
"to_hdf",
"to_stata",
}
# Third pass: extract I/O operations using current_dir
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
local_file = _resolve_python_import(alias.name, import_dir)
if local_file:
inputs.append(local_file)
elif isinstance(node, ast.ImportFrom):
if node.module and node.level == 0:
local_file = _resolve_python_import(node.module, import_dir)
if local_file:
inputs.append(local_file)
elif node.level > 0:
parent_dir = import_dir
for _ in range(node.level - 1):
parent_dir = os.path.dirname(parent_dir)
if node.module:
module_path = node.module.replace(".", os.sep)
local_file = _resolve_python_import(
module_path, parent_dir
)
if local_file:
inputs.append(local_file)
elif isinstance(node, ast.Call):
if isinstance(node.func, ast.Name):
func_name = node.func.id
if func_name == "open" and len(node.args) >= 1:
path = _resolve_variable_in_call(node.args[0], variables)
if path:
mode = "r"
if len(node.args) >= 2:
mode_str = _extract_string_from_node(node.args[1])
if mode_str:
mode = mode_str
for keyword in node.keywords:
if keyword.arg == "mode":
mode_str = _extract_string_from_node(
keyword.value
)
if mode_str:
mode = mode_str
rel_path = resolve_to_root(path)
if "w" in mode or "a" in mode or "x" in mode:
outputs.append(rel_path)
else:
inputs.append(rel_path)
elif isinstance(node.func, ast.Attribute):
module = getattr(node.func.value, "id", None)
if module and isinstance(module, str):
func = node.func.attr
if func in output_methods:
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
outputs.append(resolve_to_root(path))
continue
if module == "pd" and func.startswith("read_"):
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
inputs.append(resolve_to_root(path))
elif module in [
"df",
"data",
"frame",
"result",
] and func.startswith("to_"):
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
outputs.append(resolve_to_root(path))
elif module == "np" and func in [
"load",
"loadtxt",
"genfromtxt",
"fromfile",
]:
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
inputs.append(resolve_to_root(path))
elif module == "np" and func in [
"save",
"savetxt",
"savez",
"savez_compressed",
]:
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
outputs.append(resolve_to_root(path))
elif func == "savefig":
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
outputs.append(resolve_to_root(path))
# Handle chained method calls like ax.get_figure().savefig()
# where node.func.value is itself a Call node
elif isinstance(node.func.value, ast.Call):
func = node.func.attr
if func in output_methods:
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
outputs.append(resolve_to_root(path))
continue
if func == "savefig":
if len(node.args) >= 1:
path = _resolve_variable_in_call(
node.args[0], variables
)
if path:
outputs.append(resolve_to_root(path))
inputs = [p for p in inputs if _is_valid_project_path(p)]
outputs = [p for p in outputs if _is_valid_project_path(p)]
inputs = list(dict.fromkeys(inputs))
outputs = list(dict.fromkeys(outputs))
return {"inputs": inputs, "outputs": outputs}
def _detect_julia_code_io(
code: str, script_dir: str = "."
) -> dict[str, list[str]]:
"""Detect I/O from Julia code string (used for notebook cells)."""
inputs = []
outputs = []
code = re.sub(r"#.*$", "", code, flags=re.MULTILINE)
julia_vars = _extract_julia_string_assignments(code)
include_pattern = r'include\s*\(\s*["\']([^"\']+\.jl)["\']\s*\)'
include_matches = re.findall(include_pattern, code)
for match in include_matches:
if not os.path.isabs(match):
full_path = os.path.join(script_dir, match)
if os.path.exists(full_path):
inputs.append(Path(os.path.relpath(full_path)).as_posix())
elif _is_valid_project_path(match):
inputs.append(match)
read_patterns = [
r'open\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))\s*,\s*["\']r["\']',
r'open\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))\s*\)',
r'CSV\.(?:read|File)\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'readdlm\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
]
write_patterns = [
r'open\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))\s*,\s*["\']w["\']',
r'CSV\.write\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
r'writedlm\s*\(\s*(?:["\']([^"\']+)["\']|([A-Za-z_][A-Za-z0-9_]*))',
]
for pattern in read_patterns:
matches = re.findall(pattern, code)
for match in matches:
resolved = _resolve_julia_path_expr(match, julia_vars)
if resolved:
inputs.append(resolved)
for pattern in write_patterns:
matches = re.findall(pattern, code)
for match in matches:
resolved = _resolve_julia_path_expr(match, julia_vars)
if resolved:
outputs.append(resolved)
inputs = [p for p in inputs if _is_valid_project_path(p)]
outputs = [p for p in outputs if _is_valid_project_path(p)]
return {"inputs": inputs, "outputs": outputs}
def _resolve_python_import(
module_name: str, search_dir: str = "."
) -> str | None:
"""Resolve a Python import to a local file path if it exists."""
module_path = module_name.replace(".", os.sep)
file_path = os.path.join(search_dir, module_path + ".py")
if os.path.exists(file_path) and _is_valid_project_path(
os.path.relpath(file_path)
):
return Path(os.path.relpath(file_path)).as_posix()
init_path = os.path.join(search_dir, module_path, "__init__.py")
if os.path.exists(init_path) and _is_valid_project_path(
os.path.relpath(init_path)
):
return Path(os.path.relpath(init_path)).as_posix()
return None
def _detect_r_code_io(
code: str,
script_dir: str = ".",
wdir: str = ".",
project_root: str = ".",
_seen: set[str] | None = None,
) -> dict[str, list[str]]:
"""Detect I/O from R code string (used for scripts and notebook cells).
Parameters
----------
code : str
R code content to analyze.
script_dir : str
Directory containing the script (for resolving source() includes).
wdir : str
Working directory from which the script is executed (for data file paths).
project_root : str
Directory that ``here::here()`` resolves against, i.e. the project
root where the pipeline runs.
_seen : set[str] | None
Absolute paths of scripts already analyzed, used to guard against
cycles when recursing into ``source()``d scripts.
Returns
-------
dict[str, list[str]]
Dictionary with 'inputs' and 'outputs' keys.
"""
inputs = []
outputs = []
if _seen is None:
_seen = set()
code = re.sub(r"#.*$", "", code, flags=re.MULTILINE)
r_vars = _extract_r_string_assignments(code)
fig_dir = r_vars.get("fig_dir")
# A path argument may be a here::here()/file.path() call, a quoted literal,
# or a variable name that resolves to one of those
path_expr = (
r'(here::here\([^)]*\)|file\.path\([^)]*\)|"[^"]+"|\'[^\']+\'|'
r"[A-Za-z_][A-Za-z0-9_]*)"
)
# Detect source() calls for R script includes, add them as inputs, and
# recurse into them so their I/O is attributed to this stage
for raw in re.findall(r"source\s*\(\s*" + path_expr, code):
resolved = _resolve_r_path_expr(raw, r_vars)
if not resolved or not resolved.lower().endswith(".r"):
continue
if os.path.isabs(resolved):
continue
# here::here() paths are relative to the project root; everything else
# is resolved relative to the sourcing script's directory
if raw.strip().startswith("here::here"):
fs_path = os.path.join(project_root, resolved)
else:
fs_path = os.path.join(script_dir, resolved)
if os.path.exists(fs_path):
inputs.append(Path(os.path.relpath(fs_path)).as_posix())
elif _is_valid_project_path(resolved):
inputs.append(resolved)
# Recurse into the sourced script if we can read it
abs_path = os.path.abspath(fs_path)
if os.path.exists(fs_path) and abs_path not in _seen:
_seen.add(abs_path)
try:
with open(fs_path, "r", encoding="utf-8") as f:
sub_code = f.read()
except (OSError, UnicodeDecodeError):
sub_code = None
if sub_code is not None:
sub_dir = (
os.path.dirname(Path(os.path.relpath(fs_path)).as_posix())
or "."
)
sub_io = _detect_r_code_io(
sub_code,
script_dir=sub_dir,
wdir=wdir,
project_root=project_root,
_seen=_seen,
)
inputs.extend(sub_io["inputs"])
outputs.extend(sub_io["outputs"])
# Read patterns that capture variables, here::here()/file.path()
# expressions, or literals
read_funcs = [
r"read\.(?:csv|table|delim|tsv)",
r"readRDS",
r"readLines",
r"load",
r"read_csv",
r"read_tsv",
r"read_delim",
r"read_excel",
r"fread",
]
for func in read_funcs:
for match in re.findall(func + r"\s*\(\s*" + path_expr, code):
resolved = _resolve_r_path_expr(match, r_vars)
if resolved:
inputs.append(resolved)
# Write functions where the path is the second positional argument
second_arg_write_funcs = [
r"write\.(?:csv|table)",
r"saveRDS",
r"write_csv",
r"write_tsv",
r"write_delim",
r"write_excel\w*",
r"writeLines",
r"fwrite",
]
for func in second_arg_write_funcs:
pattern = func + r"\s*\(\s*[^,]+,\s*" + path_expr
for match in re.findall(pattern, code):
resolved = _resolve_r_path_expr(match, r_vars)
if resolved:
outputs.append(resolved)
# Write functions where the path is the (only) first positional argument
first_arg_write_funcs = [
r"ggsave",
r"sink",
r"pdf",
r"png",
r"svg",
r"svglite",
r"jpeg",
r"tiff",
r"bmp",
r"cairo_pdf",
r"CairoPNG",
r"CairoPDF",
]
for func in first_arg_write_funcs:
pattern = func + r"\s*\(\s*" + path_expr
for match in re.findall(pattern, code):
resolved = _resolve_r_path_expr(match, r_vars)
if resolved:
outputs.append(resolved)
# Write functions whose path is passed as a named argument, regardless of
# position (e.g. saveRDS(x, file = ...), pdf(file = ...), save(file = ...))
named_arg_write_funcs = [
r"write\.(?:csv|table)",
r"saveRDS",
r"save",
r"write_csv",
r"write_tsv",
r"write_delim",
r"write_excel\w*",
r"writeLines",
r"fwrite",
r"ggsave",
r"sink",
r"pdf",
r"png",
r"svg",
r"svglite",
r"jpeg",
r"tiff",
r"bmp",
r"cairo_pdf",
r"CairoPNG",
r"CairoPDF",
]
for func in named_arg_write_funcs:
pattern = (
func + r"\s*\([^)]*\b(?:file|filename|path|con)\s*=\s*" + path_expr
)
for match in re.findall(pattern, code):
resolved = _resolve_r_path_expr(match, r_vars)
if resolved:
outputs.append(resolved)
# save_fig() writes into fig_dir when the path isn't already qualified
save_fig_patterns = [
r"save_fig\s*\(\s*[^,]+,\s*" + path_expr,
r"save_fig\s*\([^)]*filename\s*=\s*" + path_expr,
]
for pattern in save_fig_patterns:
for match in re.findall(pattern, code):
resolved = _resolve_r_path_expr(match, r_vars)
if resolved:
if fig_dir and not os.path.isabs(resolved):
fig_dir_norm = fig_dir.rstrip("/\\")
if not resolved.startswith(
(fig_dir_norm + os.sep, fig_dir_norm + "/")
):
resolved = os.path.join(fig_dir, resolved)
outputs.append(resolved)
inputs = [p for p in inputs if _is_valid_project_path(p)]
outputs = [p for p in outputs if _is_valid_project_path(p)]
# Resolve paths relative to working directory
inputs = _resolve_paths_to_wdir(inputs, script_dir, wdir)
outputs = _resolve_paths_to_wdir(outputs, script_dir, wdir)
# Normalize to POSIX separators for persistence/display, since paths built
# from here::here()/file.path() use the host OS separator
inputs = [Path(p).as_posix() for p in inputs]
outputs = [Path(p).as_posix() for p in outputs]
inputs = list(dict.fromkeys(inputs))
outputs = list(dict.fromkeys(outputs))
# A file produced within this stage is not an external input, even if it is
# also read back (e.g. an intermediate written by one sourced script and
# consumed by another)
output_set = set(outputs)
inputs = [p for p in inputs if p not in output_set]
return {"inputs": inputs, "outputs": outputs}
def _extract_r_string_assignments(code: str) -> dict[str, str]:
"""Extract simple string assignments in R code.
Captures both string-literal assignments (``x <- "path"``) and
``here::here()``/``file.path()`` assignments whose arguments are literals
or previously assigned variables (``x <- here::here("a", "b")``).
"""
assignments = {}
pattern = re.compile(
r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:<-|=)\s*[\"']([^\"']+)[\"']",
flags=re.MULTILINE,
)
for name, value in pattern.findall(code):
assignments[name] = value
# Resolve here::here()/file.path() assignments using the literals above
call_pattern = re.compile(
r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:<-|=)\s*"
r"(here::here\([^)]*\)|file\.path\([^)]*\))",
flags=re.MULTILINE,
)
for name, expr in call_pattern.findall(code):
resolved = _resolve_r_path_expr(expr, assignments)
if resolved:
assignments[name] = resolved
return assignments
def _extract_r_string_vector_assignments(code: str) -> dict[str, list[str]]:
"""Extract character-vector assignments in R code.
Captures assignments of the form ``x <- c("a", "b")`` so that a later
``pacman::p_load(char = x)`` can be resolved back to its package list.
"""
assignments = {}
pattern = re.compile(
r"([A-Za-z_][A-Za-z0-9_.]*)\s*(?:<-|=)\s*c\(([^)]*)\)",
flags=re.DOTALL,
)
for name, body in pattern.findall(code):
strings = re.findall(r'["\']([a-zA-Z0-9._]+)["\']', body)
if strings:
assignments[name] = strings
return assignments
def _resolve_r_package_args(
call_args: str, vector_assignments: dict[str, list[str]]
) -> set[str]:
"""Resolve package names from a ``pacman::p_load()`` argument list.
Handles quoted literals, inline ``c(...)`` vectors, bare (non-standard
evaluation) names like ``p_load(dplyr, tidyr)``, and ``char = x`` where
``x`` is a character vector assigned earlier in the script.
"""
packages = set()
# Quoted strings are package names, whether passed directly or inside c()
packages.update(re.findall(r'["\']([a-zA-Z0-9._]+)["\']', call_args))
# char = <var> referencing a character-vector assignment
for var in re.findall(r"char\s*=\s*([A-Za-z_][A-Za-z0-9_.]*)", call_args):
packages.update(vector_assignments.get(var, []))
# Bare names passed positionally; drop keyword arguments (install=, update=,
# character.only=, etc.) and their values before collecting identifiers
stripped = re.sub(r"[A-Za-z_][A-Za-z0-9_.]*\s*=\s*[^,]+", "", call_args)
for token in re.findall(r"[A-Za-z_][A-Za-z0-9_.]*", stripped):
# c() is the vector constructor, not a package; TRUE/FALSE are values
if token not in {"c", "TRUE", "FALSE", "T", "F"}:
packages.add(token)
return packages
def _extract_julia_string_assignments(code: str) -> dict[str, str]:
"""Extract simple string assignments in Julia code."""
assignments = {}
pattern = re.compile(
r"^\s*(?:const\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*[\"']([^\"']+)[\"']",
flags=re.MULTILINE,
)
for name, value in pattern.findall(code):
assignments[name] = value
return assignments
def _resolve_julia_path_expr(
match: tuple[str, str] | str,
variables: dict[str, str],
) -> str | None:
"""Resolve a Julia path expression from a regex match."""
if isinstance(match, tuple):
literal, var = match
if literal:
return literal
if var:
return variables.get(var)
return None
return variables.get(match) or match
def _resolve_r_path_expr(expr: str, variables: dict[str, str]) -> str | None:
"""Resolve simple R path expressions.
Handles quoted literals, variables, and ``file.path()``/``here::here()``
calls whose arguments are literals or known variables.
"""
expr = expr.strip()
if not expr:
return None
if expr[0] in ['"', "'"] and expr[-1] == expr[0]:
return expr[1:-1]
if expr in variables:
return variables[expr]
for prefix in ("file.path", "here::here"):
if expr.startswith(prefix):
match = re.match(re.escape(prefix) + r"\((.*)\)", expr)
if not match:
return None
args = [a.strip() for a in match.group(1).split(",") if a.strip()]
if not args:
return None
parts = []
for arg in args:
if arg[0] in ['"', "'"] and arg[-1] == arg[0]:
parts.append(arg[1:-1])
elif arg in variables:
parts.append(variables[arg])
else:
return None
return os.path.join(*parts)
return None
def generate_stage_name(cmd: list[str]) -> str:
"""Generate a stage name from a command.
Parameters
----------
cmd : list[str]
The full command as a list of strings.
Returns
-------
str
The generated stage name.
"""
if not cmd:
return "stage"
first_arg = cmd[0]
# Check if first arg is a script/notebook file
script_extensions = [
".py",
".jl",
".m",
".ipynb",
".tex",
".sh",
".bash",
".zsh",
".R",
]
if any(first_arg.endswith(ext) for ext in script_extensions):
# Extract base name from the script/notebook path
base_name = os.path.splitext(os.path.basename(first_arg))[0]
stage_name = base_name
elif first_arg == "python" and len(cmd) > 1 and cmd[1].endswith(".py"):
# Special handling for python script.py - use the script
# filename as base
script_path = cmd[1]
base_name = os.path.splitext(os.path.basename(script_path))[0]
stage_name = base_name
elif first_arg == "julia" and len(cmd) > 1 and cmd[1].endswith(".jl"):
# Special handling for julia script.jl - use the script
# filename as base
script_path = cmd[1]
base_name = os.path.splitext(os.path.basename(script_path))[0]
stage_name = base_name
elif first_arg == "matlab" and len(cmd) > 1 and cmd[1].endswith(".m"):
# Special handling for matlab script.m - use the script
# filename as base
script_path = cmd[1]
base_name = os.path.splitext(os.path.basename(script_path))[0]
stage_name = base_name
elif first_arg == "Rscript" and len(cmd) > 1 and cmd[1].endswith(".R"):
# Special handling for Rscript - use the script filename as base
script_path = cmd[1]
base_name = os.path.splitext(os.path.basename(script_path))[0]
stage_name = base_name
else:
# For commands (like "echo", "matlab", "julia", etc.)
# Use the command name as the base
stage_name = first_arg
# Shell interpreters should not include their arguments in the name
# (as those often contain inline scripts with special characters)
if first_arg not in ["bash", "sh", "zsh"] and len(cmd) > 1:
# For MATLAB commands, filter out -batch flag
remaining_args = cmd[1:]
if first_arg == "matlab" and "-batch" in remaining_args:
remaining_args = [
arg for arg in remaining_args if arg != "-batch"
]
if remaining_args:
args_part = "-".join(remaining_args)
stage_name += "-" + args_part
# Convert to kebab-case
stage_name = stage_name.replace("_", "-").lower()
# Replace dots with dashes (except file extensions which are
# already stripped)
stage_name = stage_name.replace(".", "-")
# Replace spaces with dashes
stage_name = stage_name.replace(" ", "-")
# Remove parentheses, slashes and other special characters
# (including shell redirects)
stage_name = re.sub(r"[(){}\[\]'\"><|&;/]", "", stage_name)
# Consolidate multiple dashes into single dashes
stage_name = re.sub(r"-+", "-", stage_name)
# Remove leading/trailing dashes
stage_name = stage_name.strip("-")
# If any args include the .ipynb extension, add a -notebook suffix
if any(arg.endswith(".ipynb") for arg in cmd):
stage_name += "-notebook"
return stage_name
def detect_io(stage: dict) -> dict[str, list[str]]:
"""Detect inputs and outputs for a stage based on its kind.
Parameters
----------
stage : dict
Stage configuration dictionary with 'kind', 'environment', and other
stage-specific fields.
Returns
-------
dict[str, list[str]]
Dictionary with 'inputs' and 'outputs' keys containing detected paths.
"""
from calkit.matlab import detect_matlab_command_io, detect_matlab_script_io
stage_kind = stage.get("kind")
script_path = (
stage.get("script_path")
or stage.get("notebook_path")
or stage.get("target_path")
)
environment = stage.get("environment", "_system")
wdir = stage.get("wdir")
if stage_kind == "python-script" and script_path:
return detect_python_script_io(script_path, wdir=wdir)
elif stage_kind == "r-script" and script_path:
return detect_r_script_io(script_path, wdir=wdir)
elif stage_kind == "julia-script" and script_path:
return detect_julia_script_io(script_path, wdir=wdir)
elif stage_kind == "shell-script" and script_path:
return detect_shell_script_io(script_path, wdir=wdir)
elif stage_kind == "jupyter-notebook" and script_path:
return detect_jupyter_notebook_io(script_path, wdir=wdir)
elif stage_kind == "latex" and script_path:
return detect_latex_io(script_path)
elif stage_kind == "matlab-script" and script_path:
return detect_matlab_script_io(
script_path, environment=environment, wdir=wdir
)
elif stage_kind == "matlab-command":
command = stage.get("command", "")
wdir_str = wdir if wdir else "."
return detect_matlab_command_io(
command, environment=environment, wdir=wdir_str
)
elif stage_kind == "shell-command":
command = stage.get("command", "")
return detect_shell_command_io(command)
# Return empty results for unsupported or unrecognized stage kinds
return {"inputs": [], "outputs": []}
def _is_stdlib_module(module_name: str) -> bool:
"""Check if a module is part of the Python standard library."""
# Get the base module name (before any dots)
base_module = module_name.split(".")[0]
# Known stdlib modules that might not be detected by other methods
stdlib_modules = (
set(sys.stdlib_module_names)
if hasattr(sys, "stdlib_module_names")
else set()
)
if base_module in stdlib_modules:
return True
# Additional check for common stdlib modules
common_stdlib = {
"os",
"sys",
"re",
"json",
"math",
"datetime",
"time",
"random",
"collections",
"itertools",
"functools",
"pathlib",
"subprocess",
"typing",
"io",
"copy",
"pickle",
"csv",
"sqlite3",
"unittest",
"logging",
"argparse",
"configparser",
"email",
"urllib",
"http",
"html",
"xml",
"hashlib",
"base64",
"tempfile",
"shutil",
"glob",
"fnmatch",
"contextlib",
"abc",
"dataclasses",
"enum",
"decimal",
"fractions",
"statistics",
"platform",
"socket",
"ssl",
"asyncio",
"concurrent",
"multiprocessing",
"threading",
"queue",
"warnings",
}
return base_module in common_stdlib
_PYTHON_IMPORT_TO_DISTRIBUTION = {
"sklearn": "scikit-learn",
"skimage": "scikit-image",
"cv2": "opencv-python",
"PIL": "Pillow",
"bs4": "beautifulsoup4",
"yaml": "PyYAML",
"dateutil": "python-dateutil",
"dotenv": "python-dotenv",
"serial": "pyserial",
"git": "GitPython",
}
def detect_python_dependencies(
script_path: str | None = None,
code: str | None = None,
) -> list[str]:
"""Detect non-stdlib dependencies from a Python script or code string.
Parameters
----------
script_path : str | None
Path to Python script. Either this or code must be provided.
code : str | None
Python code string. Either this or script_path must be provided.
Returns
-------
list[str]
List of non-stdlib package names.
"""
if script_path is None and code is None:
raise ValueError("Either script_path or code must be provided")
if code is None:
assert script_path is not None # Type guard
if not os.path.exists(script_path):
return []
try:
with open(script_path, "r", encoding="utf-8") as f:
code = f.read()
except (UnicodeDecodeError, IOError):
return []
assert code is not None # Type guard
dependencies = set()
# Strip IPython magic commands (line magics like %matplotlib and
# cell magics like %%timeit) as they are not valid Python syntax
lines = code.split("\n")
cleaned_lines = []
in_cell_magic = False
for line in lines:
stripped = line.strip()
# Check for cell magic start (%%...)
if stripped.startswith("%%"):
in_cell_magic = True
continue
# If we're in a cell magic block, skip until we hit a blank line
# or a line that doesn't look like it's part of the magic
if in_cell_magic:
if stripped == "" or not line.startswith((" ", "\t")):
in_cell_magic = False
else:
continue
# Skip line magics (%...)
if stripped.startswith("%"):
continue
cleaned_lines.append(line)
cleaned_code = "\n".join(cleaned_lines)
try:
tree = ast.parse(cleaned_code)
except SyntaxError:
return []
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
module_name = alias.name
if not _is_stdlib_module(module_name):
# Get the top-level package name
top_level = module_name.split(".")[0]
dependencies.add(
_PYTHON_IMPORT_TO_DISTRIBUTION.get(
top_level, top_level
)
)
elif isinstance(node, ast.ImportFrom):
if node.module and node.level == 0:
module_name = node.module
if not _is_stdlib_module(module_name):
top_level = module_name.split(".")[0]
dependencies.add(
_PYTHON_IMPORT_TO_DISTRIBUTION.get(
top_level, top_level
)
)
return sorted(list(dependencies))
def detect_r_dependencies(
script_path: str | None = None,
code: str | None = None,
) -> list[str]:
"""Detect package dependencies from an R script or code string.
Parameters
----------
script_path : str | None
Path to R script. Either this or code must be provided.
code : str | None
R code string. Either this or script_path must be provided.
Returns
-------
list[str]
List of R package names.
"""
if script_path is None and code is None:
raise ValueError("Either script_path or code must be provided")
if code is None:
assert script_path is not None # Type guard
if not os.path.exists(script_path):
return []
try:
with open(script_path, "r", encoding="utf-8") as f:
code = f.read()
except (UnicodeDecodeError, IOError):
return []
assert code is not None # Type guard
dependencies = set()
# Remove comments
code = re.sub(r"#.*$", "", code, flags=re.MULTILINE)
# Patterns for library() and require() calls
patterns = [
r'library\s*\(\s*["\']?([a-zA-Z0-9._]+)["\']?\s*\)',
r'require\s*\(\s*["\']?([a-zA-Z0-9._]+)["\']?\s*\)',
]
for pattern in patterns:
matches = re.findall(pattern, code)
dependencies.update(matches)
# requireNamespace()/loadNamespace() take the package as a string and are
# commonly used to guard optional installs (e.g. pacman/here bootstrapping)
dependencies.update(
re.findall(
r'(?:require|load)Namespace\s*\(\s*["\']([a-zA-Z0-9._]+)["\']',
code,
)
)
# Qualified calls like pkg::fn() or pkg:::fn() reference a package without
# ever attaching it, so the package itself is still a dependency
dependencies.update(re.findall(r"\b([a-zA-Z][a-zA-Z0-9._]*)\s*:::?", code))
# pacman::p_load() loads (and optionally installs) packages dynamically; its
# arguments may be bare names, quoted strings, or a character vector passed
# via char=, possibly referencing a variable assigned earlier in the script
vector_assignments = _extract_r_string_vector_assignments(code)
for call_args in re.findall(
r"p_load\s*\(((?:[^()]|\([^()]*\))*)\)", code, flags=re.DOTALL
):
dependencies.update(
_resolve_r_package_args(call_args, vector_assignments)
)
# Filter out base R packages
base_packages = {
"base",
"compiler",
"datasets",
"graphics",
"grDevices",
"grid",
"methods",
"parallel",
"splines",
"stats",
"stats4",
"tcltk",
"tools",
"utils",
}
dependencies = {pkg for pkg in dependencies if pkg not in base_packages}
return sorted(list(dependencies))
def detect_julia_dependencies(
script_path: str | None = None,
code: str | None = None,
) -> list[str]:
"""Detect package dependencies from a Julia script or code string.
Parameters
----------
script_path : str | None
Path to Julia script. Either this or code must be provided.
code : str | None
Julia code string. Either this or script_path must be provided.
Returns
-------
list[str]
List of Julia package names.
"""
if script_path is None and code is None:
raise ValueError("Either script_path or code must be provided")
if code is None:
assert script_path is not None # Type guard
if not os.path.exists(script_path):
return []
try:
with open(script_path, "r", encoding="utf-8") as f:
code = f.read()
except (UnicodeDecodeError, IOError):
return []
assert code is not None # Type guard
dependencies = set()
# Remove comments
code = re.sub(r"#.*$", "", code, flags=re.MULTILINE)
# Pattern for using statements
pattern = r"using\s+([a-zA-Z0-9._]+)"
matches = re.findall(pattern, code)
dependencies.update(matches)
return sorted(list(dependencies))
def detect_dependencies_from_notebook(
notebook_path: str,
language: NotebookLanguage | None = None,
) -> list[str]:
"""Detect dependencies from a Jupyter notebook.
Parameters
----------
notebook_path : str
Path to the notebook.
language : Literal["python", "julia", "r"] | None
Language of the notebook. If None, will be detected from metadata.
Returns
-------
list[str]
List of package/module names.
"""
if not os.path.exists(notebook_path):
return []
try:
with open(notebook_path, "r", encoding="utf-8") as f:
nb = json.load(f)
except (json.JSONDecodeError, UnicodeDecodeError):
return []
# Detect language if not provided
if language is None:
detected = language_from_notebook(notebook_path, notebook=nb)
language = detected if detected is not None else "python"
# Collect all code from cells
code_cells = [
cell for cell in nb.get("cells", []) if cell.get("cell_type") == "code"
]
all_code = []
for cell in code_cells:
source = cell.get("source", [])
if isinstance(source, list):
all_code.append("".join(source))
else:
all_code.append(source)
combined_code = "\n".join(all_code)
# Detect dependencies based on language
if language == "python":
return detect_python_dependencies(code=combined_code)
elif language == "julia":
return detect_julia_dependencies(code=combined_code)
elif language == "r":
return detect_r_dependencies(code=combined_code)
return []
def create_python_requirements_file(
dependencies: list[str],
output_path: str,
) -> None:
"""Create a requirements.txt file from a list of dependencies.
Parameters
----------
dependencies : list[str]
List of package names.
output_path : str
Path where the requirements.txt should be created.
"""
from calkit.environments import create_python_requirements_content
output_dir = os.path.dirname(output_path)
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(output_path, "w") as f:
f.write(create_python_requirements_content(dependencies))
def create_julia_project_file(
dependencies: list[str],
output_path: str,
project_name: str = "environment",
) -> None:
"""Create a Julia Project.toml file from a list of dependencies.
Parameters
----------
dependencies : list[str]
List of package names.
output_path : str
Path where the Project.toml should be created.
project_name : str
Name of the Julia project.
"""
from calkit.environments import create_julia_project_file_content
output_dir = os.path.dirname(output_path)
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(output_path, "w") as f:
f.write(create_julia_project_file_content(dependencies, project_name))
def create_r_description_file(
dependencies: list[str],
output_path: str,
) -> None:
"""Create a simple R DESCRIPTION file listing dependencies.
This creates a minimal DESCRIPTION file that renv can work with.
Parameters
----------
dependencies : list[str]
List of R package names.
output_path : str
Path where the DESCRIPTION should be created.
"""
from calkit.environments import create_r_description_content
output_dir = os.path.dirname(output_path)
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(output_path, "w") as f:
f.write(create_r_description_content(dependencies))
# Figure/dataset auto-detection
# A file is only treated as an auto-detected figure or dataset when it lives in
# a directory whose name signals its kind. These sets are intentionally narrow
# (and kept in sync with the Calkit hub) so we don't flag arbitrary images or
# data files scattered around a repository.
FIGURE_EXTENSIONS = {
".png",
".jpg",
".jpeg",
".gif",
".svg",
".pdf",
".eps",
".tiff",
".tif",
}
FIGURE_DIRS = {"figures", "figure", "figs", "fig", "plots", "plot", "images"}
DATASET_EXTENSIONS = {
".csv",
".h5",
".hdf5",
".parquet",
".nc",
".zarr",
".feather",
".arrow",
".avro",
".json",
".jsonl",
".ndjson",
}
DATA_DIRS = {"data", "datasets", "dataset"}
RESULT_DIRS = {"results", "result"}
RESULT_EXTENSIONS = {
".json",
".csv",
".tsv",
".yaml",
".yml",
".parquet",
".h5",
".hdf5",
".txt",
".html",
}
PRESENTATION_DIRS = {"presentations", "presentation", "slides", "talks"}
PRESENTATION_EXTENSIONS = {".pdf", ".pptx", ".key"}
# Files that look like a presentation regardless of which directory they live in.
PRESENTATION_NAMES = {
"slides.pdf",
"slides.pptx",
"slides.key",
"presentation.pdf",
"presentation.pptx",
"presentation.key",
"talk.pdf",
}
DETECTION_IGNORE_FPATH = os.path.join(".calkit", "ignore")
def load_detection_ignore(wdir: str | None = None) -> pathspec.PathSpec | None:
"""Load ``.calkit/ignore`` artifact detection exclusions."""
import pathspec
fpath = DETECTION_IGNORE_FPATH
if wdir is not None:
fpath = os.path.join(wdir, fpath)
if not os.path.isfile(fpath):
return None
with open(fpath, encoding="utf-8") as f:
return pathspec.PathSpec.from_lines("gitwildmatch", f)
def _ancestor_dir_names(rel_path: str) -> set[str]:
"""Lower-cased names of the ancestor directories of a "/"-separated path."""
return {p.lower() for p in rel_path.split("/")[:-1]}
def _path_ext(rel_path: str) -> str:
name = rel_path.rsplit("/", 1)[-1]
return ("." + name.rsplit(".", 1)[-1].lower()) if "." in name else ""
def is_figure_path(
rel_path: str, *, ignore: pathspec.PathSpec | None = None
) -> bool:
"""Whether a repo-relative path looks like an auto-detectable figure."""
if ignore is not None and ignore.match_file(rel_path):
return False
ancestors = _ancestor_dir_names(rel_path)
ext = _path_ext(rel_path)
if ext in FIGURE_EXTENSIONS and ancestors & FIGURE_DIRS:
return True
# A Plotly figure is stored as JSON; treat .json under a figure-only
# directory (one that isn't also a data directory) as a figure.
if ext == ".json" and ancestors & (FIGURE_DIRS - DATA_DIRS):
return True
return False
def is_dataset_path(
rel_path: str, *, ignore: pathspec.PathSpec | None = None
) -> bool:
"""Whether a repo-relative path looks like an auto-detectable dataset."""
if ignore is not None and ignore.match_file(rel_path):
return False
return (
_path_ext(rel_path) in DATASET_EXTENSIONS
and bool(_ancestor_dir_names(rel_path) & DATA_DIRS)
and not is_figure_path(rel_path, ignore=ignore)
)
def is_result_path(
rel_path: str, *, ignore: pathspec.PathSpec | None = None
) -> bool:
"""Whether a repo-relative path looks like an auto-detectable result.
A result is a data-like file (JSON, CSV, etc.) under a ``results``-style
directory. Anything already detected as a figure or dataset is excluded.
"""
if ignore is not None and ignore.match_file(rel_path):
return False
return (
_path_ext(rel_path) in RESULT_EXTENSIONS
and bool(_ancestor_dir_names(rel_path) & RESULT_DIRS)
and not is_figure_path(rel_path, ignore=ignore)
and not is_dataset_path(rel_path, ignore=ignore)
)
def is_presentation_path(
rel_path: str, *, ignore: pathspec.PathSpec | None = None
) -> bool:
"""Whether a repo-relative path looks like an auto-detectable presentation.
Either a slide-deck file (PDF/PPTX/KEY) under a ``slides``/``presentations``
directory, or a file with a presentation-like name (e.g. ``slides.pdf``,
``presentation.pdf``) anywhere. Figures are excluded.
"""
if ignore is not None and ignore.match_file(rel_path):
return False
if is_figure_path(rel_path, ignore=ignore):
return False
name = rel_path.rsplit("/", 1)[-1].lower()
if name in PRESENTATION_NAMES:
return True
return _path_ext(rel_path) in PRESENTATION_EXTENSIONS and bool(
_ancestor_dir_names(rel_path) & PRESENTATION_DIRS
)
PUBLICATION_DIRS = {
"publication",
"publications",
"paper",
"papers",
"manuscript",
"manuscripts",
"thesis",
"report",
"reports",
"pub",
"pubs",
}
PUBLICATION_EXTENSIONS = {".pdf", ".tex", ".docx"}
# Files that look like a publication regardless of which directory they live in.
PUBLICATION_NAMES = {
"paper.pdf",
"manuscript.pdf",
"thesis.pdf",
"report.pdf",
"paper.tex",
"main.tex",
"manuscript.tex",
"thesis.tex",
}
def is_publication_path(
rel_path: str, *, ignore: pathspec.PathSpec | None = None
) -> bool:
"""Whether a repo-relative path looks like an auto-detectable publication.
Either a document (PDF/TeX/DOCX) under a ``paper``/``publication``/
``thesis``-style directory, or a file with a publication-like name (e.g.
``manuscript.pdf``, ``main.tex``) anywhere. Figures and presentations are
excluded.
"""
if ignore is not None and ignore.match_file(rel_path):
return False
if is_figure_path(rel_path, ignore=ignore) or is_presentation_path(
rel_path, ignore=ignore
):
return False
name = rel_path.rsplit("/", 1)[-1].lower()
if name in PUBLICATION_NAMES:
return True
return _path_ext(rel_path) in PUBLICATION_EXTENSIONS and bool(
_ancestor_dir_names(rel_path) & PUBLICATION_DIRS
)
def detect_artifact_kind(
rel_path: str, *, ignore: pathspec.PathSpec | None = None
) -> str | None:
"""Infer an artifact's release kind from its repo-relative path.
Returns one of ``"figure"``, ``"presentation"``, ``"publication"``, or
``"dataset"``, or ``None`` if the path doesn't look like any of these.
"""
if is_figure_path(rel_path, ignore=ignore):
return "figure"
if is_presentation_path(rel_path, ignore=ignore):
return "presentation"
if is_publication_path(rel_path, ignore=ignore):
return "publication"
if is_dataset_path(rel_path, ignore=ignore):
return "dataset"
return None
def _is_hidden_path(rel_path: str) -> bool:
return any(part.startswith(".") for part in rel_path.split("/"))
def _is_under_any_dir(rel_path: str, dirs: list[str]) -> bool:
"""Whether ``rel_path`` equals or lives inside any of ``dirs``."""
return any(rel_path == d or rel_path.startswith(d + "/") for d in dirs)
def _collapse_dataset_folders(paths: list[str]) -> list[str]:
"""Collapse data files into their containing folder.
A folder holding two or more data files is reported as a single dataset
rather than file-by-file. Folders nested inside another collapsed folder
are dropped so only the outermost dataset folder remains.
"""
by_dir: dict[str, list[str]] = {}
for p in paths:
directory = p.rsplit("/", 1)[0] if "/" in p else ""
by_dir.setdefault(directory, []).append(p)
collapsed: set[str] = set()
for directory, group in by_dir.items():
if directory and len(group) >= 2:
collapsed.add(directory)
else:
collapsed.update(group)
result = sorted(collapsed)
return [
p
for p in result
if not any(o != p and p.startswith(o + "/") for o in result)
]
def detect_figures(
candidate_paths: list[str],
reserved_paths: list[str] | tuple[str, ...] = (),
ignore: pathspec.PathSpec | None = None,
) -> list[str]:
"""Auto-detected figure paths among ``candidate_paths``.
Paths under a dot-directory or under one of ``reserved_paths`` (declared
artifacts and pipeline outputs) are skipped.
"""
reserved = list(reserved_paths)
return sorted(
{
p
for p in candidate_paths
if not _is_hidden_path(p)
and not _is_under_any_dir(p, reserved)
and is_figure_path(p, ignore=ignore)
}
)
def detect_datasets(
candidate_paths: list[str],
reserved_paths: list[str] | tuple[str, ...] = (),
figure_paths: list[str] | tuple[str, ...] = (),
ignore: pathspec.PathSpec | None = None,
) -> list[str]:
"""Auto-detected dataset paths among ``candidate_paths``.
Anything already detected as a figure (``figure_paths``) is excluded, and a
folder full of data files collapses to a single dataset entry.
"""
reserved = list(reserved_paths)
figset = set(figure_paths)
files = {
p
for p in candidate_paths
if not _is_hidden_path(p)
and not _is_under_any_dir(p, reserved)
and p not in figset
and is_dataset_path(p, ignore=ignore)
}
return _collapse_dataset_folders(sorted(files))
def detect_results(
candidate_paths: list[str],
reserved_paths: list[str] | tuple[str, ...] = (),
ignore: pathspec.PathSpec | None = None,
) -> list[str]:
"""Auto-detected result paths among ``candidate_paths``.
Paths under a dot-directory or under one of ``reserved_paths`` (declared
artifacts) are skipped.
"""
reserved = list(reserved_paths)
return sorted(
{
p
for p in candidate_paths
if not _is_hidden_path(p)
and not _is_under_any_dir(p, reserved)
and is_result_path(p, ignore=ignore)
}
)
def detect_presentations(
candidate_paths: list[str],
reserved_paths: list[str] | tuple[str, ...] = (),
ignore: pathspec.PathSpec | None = None,
) -> list[str]:
"""Auto-detected presentation paths among ``candidate_paths``."""
reserved = list(reserved_paths)
return sorted(
{
p
for p in candidate_paths
if not _is_hidden_path(p)
and not _is_under_any_dir(p, reserved)
and is_presentation_path(p, ignore=ignore)
}
)
def list_repo_files(wdir: str | None = None) -> list[str]:
"""Repo-relative paths of files Git does not ignore.
Combines tracked and untracked files while honoring ``.gitignore``, so
detection skips virtualenvs, ``node_modules``, and other ignored content.
Paths are returned relative to ``wdir`` (or the current directory), using
POSIX separators; files outside ``wdir`` are omitted.
"""
import calkit.git
try:
repo = calkit.git.get_repo(wdir)
except Exception:
return []
files = calkit.git.ls_files(
repo, "--cached", "--others", "--exclude-standard"
)
repo_root = repo.working_dir
base = os.path.abspath(wdir if wdir is not None else ".")
rel_files = []
for f in files:
rel = os.path.relpath(os.path.join(repo_root, f), base)
if rel.startswith(".."):
continue
rel_files.append(Path(rel).as_posix())
return rel_files
def list_dvc_tracked_files(wdir: str | None = None) -> list[str]:
"""Repo-relative paths of files tracked by DVC.
DVC-tracked artifacts (added via ``dvc add`` or produced as pipeline
outputs) are Git-ignored, so they don't appear in :func:`list_repo_files`;
include them so figures and datasets stored with DVC are still detected.
Returns an empty list when DVC isn't available or the project isn't a DVC
repo.
"""
from calkit.dvc.core import list_paths
try:
return [
p.replace("\\", "/")
for p in list_paths(wdir=wdir, recursive=True)
if p
]
except Exception:
return []
def _reserved_artifact_paths(
wdir: str | None = None, ck_info: dict | None = None
) -> list[str]:
"""Paths of artifacts declared in ``calkit.yaml``.
These are excluded from auto-detection so a declared figure/dataset isn't
reported twice and individual files inside a declared artifact folder aren't
listed separately. Pipeline/DVC stage outputs are intentionally *not*
reserved: a figure or dataset produced by the pipeline should still be
detected (and shown with its producing stage as provenance).
"""
import calkit
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
paths: list[str] = []
for kind in ("figures", "datasets", "results", "presentations"):
for obj in ck_info.get(kind) or []:
if isinstance(obj, dict) and isinstance(obj.get("path"), str):
paths.append(obj["path"])
return [p.replace("\\", "/") for p in paths]
def detect_project_artifacts(
wdir: str | None = None, ck_info: dict | None = None
) -> dict[str, list[str]]:
"""Auto-detect figure and dataset paths in the project at ``wdir``.
Candidates are the files Git does not ignore plus any files tracked by DVC
(which are Git-ignored); declared artifacts are reserved so they aren't
detected again.
"""
import calkit
if ck_info is None:
ck_info = calkit.load_calkit_info(wdir=wdir)
ignore = load_detection_ignore(wdir=wdir)
reserved = _reserved_artifact_paths(wdir=wdir, ck_info=ck_info)
candidates = list(
dict.fromkeys(
[*list_repo_files(wdir=wdir), *list_dvc_tracked_files(wdir=wdir)]
)
)
figures = detect_figures(
candidates, reserved_paths=reserved, ignore=ignore
)
datasets = detect_datasets(
candidates,
reserved_paths=reserved,
figure_paths=figures,
ignore=ignore,
)
results = detect_results(
candidates, reserved_paths=reserved, ignore=ignore
)
presentations = detect_presentations(
candidates, reserved_paths=reserved, ignore=ignore
)
return {
"figures": figures,
"datasets": datasets,
"results": results,
"presentations": presentations,
}
"""Functionality for detecting inputs, outputs, and dependencies from scripts,
notebooks, and commands.
"""
from __future__ import annotations
import ast
import json
import os
import re
import shlex
import sys
from collections.abc import Iterable
from pathlib import Path
from typing import Literal
import pathspec
NotebookLanguage = Literal["python", "julia", "r"]
def detect_python_script_io(
script_path: str,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from a Python script using static analysis."""
inputs = []
outputs = []
if not os.path.exists(script_path):
return {"inputs": inputs, "outputs": outputs}
try:
with open(script_path, "r", encoding="utf-8") as f:
content = f.read()
except UnicodeDecodeError:
return {"inputs": inputs, "outputs": outputs}
script_dir = os.path.dirname(script_path) if script_path else "."
effective_wdir = wdir or "."
return _detect_python_code_io(
content,
script_dir=script_dir,
working_dir=effective_wdir,
current_dir=effective_wdir,
)
def detect_r_script_io(
script_path: str,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from an R script using regex patterns.
Parameters
----------
script_path : str
Path to the R script file.
wdir : str | None
Working directory from which the script will be executed.
Defaults to current directory.
Returns
-------
dict[str, list[str]]
Dictionary with 'inputs' and 'outputs' keys.
"""
inputs = []
outputs = []
if not os.path.exists(script_path):
return {"inputs": inputs, "outputs": outputs}
try:
with open(script_path, "r", encoding="utf-8") as f:
content = f.read()
except UnicodeDecodeError:
return {"inputs": inputs, "outputs": outputs}
script_dir = os.path.dirname(script_path) if script_path else "."
effective_wdir = wdir or "."
return _detect_r_code_io(
content,
script_dir=script_dir,
wdir=effective_wdir,
project_root=effective_wdir,
)
def detect_julia_script_io(
script_path: str,
wdir: str | None = None,
) -> dict[str, list[str]]:
"""Detect inputs and outputs from a Julia script using regex patterns.
Parameters
----------
script_path : str
Path to the Julia script file.
wdir : str | None
Working directory from which the script will be executed.
Defaults to current directory.
Returns
-------
dict[str, list[str]]
Xet Storage Details
- Size:
- 85.1 kB
- Xet hash:
- f15bde4abf521e6a588e254d698e846009ab1b8d13282fdf308f6520ab7da493
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.