Buckets:

download
raw
24.6 kB
# SPDX-FileCopyrightText: 2025-present MTS PJSC
# SPDX-License-Identifier: Apache-2.0
"""
Fix unresolved doc-anchor references in converted docstrings.
Replaces two kinds of unresolved cross-references:
1. [label][] — converted from RST :ref:, resolved via REF_MAP
2. [text][syncmaster.module.ClassName] — Python-path refs that autorefs cannot
resolve (because show_root_heading: false suppresses anchor generation),
resolved via PYTHON_PATH_MAP
Both mapping tables are the single source of truth. Update them when doc pages
are renamed, reorganised, or when new unresolved references appear.
Usage:
python fix_refs.py [--dry-run] [--path PATH] [--file FILE] [--docs-path PATH]
Options:
--dry-run Print changes without writing files.
--path PATH Directory to process recursively (default: syncmaster/).
--file FILE Process a single file instead of a directory.
--docs-path PATH Also fix broken links in docs .md files under PATH.
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
# ---------------------------------------------------------------------------
# Mapping: RST :ref: label → (link text, absolute docs path)
#
# Paths are absolute from the docs root (as served by MkDocs).
# Update this table when doc pages are renamed or reorganised.
# ---------------------------------------------------------------------------
REF_MAP: dict[str, tuple[str, str]] = {
# All :ref: labels were converted manually during RST→MD migration.
# Add entries here if new label-style references appear.
}
# ---------------------------------------------------------------------------
# Mapping: Python dotted path → absolute docs URL (with anchor)
#
# Used for [text][onetl.module.ClassName] cross-references that autorefs
# cannot resolve because show_root_heading: false suppresses anchor generation
# for documented root objects.
# ---------------------------------------------------------------------------
PYTHON_PATH_MAP: dict[str, str] = {
# Add entries here for [text][syncmaster.*] refs that autorefs cannot resolve
# because show_root_heading: false suppresses the anchor, mapping to explicit URLs.
}
# ---------------------------------------------------------------------------
# Mapping: short class name label → absolute docs URL with anchor
#
# Used for [text][ShortClassName] cross-references where the label is a plain
# class name (not a full onetl.* path and not a kebab-case REF_MAP key).
# ---------------------------------------------------------------------------
LABEL_MAP: dict[str, str] = {}
# ---------------------------------------------------------------------------
# Mapping: Python dotted path → (display text, DBR anchor label)
#
# Used for [onetl.path][] cross-references where the target is documented
# via a { #DBR-... } anchor (not a mkdocstrings ::: directive).
# Output format: [display text][DBR-anchor]
# ---------------------------------------------------------------------------
ANCHOR_REF_MAP: dict[str, tuple[str, str]] = {}
# ---------------------------------------------------------------------------
# Mapping: Python dotted path → inline code name
#
# Used for [onetl.path][] cross-references where the target is NOT documented
# and cannot be linked. Output: `ClassName` (inline code, no link).
# ---------------------------------------------------------------------------
CODE_PATH_MAP: dict[str, str] = {
# Functions/classes not documented on any page → inline code (no link)
"syncmaster.server.application_factory": "`application_factory`",
}
# ---------------------------------------------------------------------------
# Mapping: Python dotted path (wrong/legacy) → plain name
#
# Used for [text][onetl.path] where the path is incorrect or the target
# should be rendered as a plain ExprName (let griffe/autorefs resolve it).
# Output: just the name, no brackets.
# ---------------------------------------------------------------------------
PLAIN_REF_MAP: dict[str, str] = {
# Wrong Python paths in docstrings → correct inline code
# (target is not documented, so just render as code)
}
# ---------------------------------------------------------------------------
# Mapping: short [Name][] refs not in onetl.* namespace
#
# Used for [Name][] where Name is a simple identifier or dotted non-onetl path.
# Values wrapped in backticks become inline code; plain strings become plain text.
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Mapping: broken relative links in docs .md files → correct replacement
#
# Used for links that cannot be resolved by MkDocs because the target file
# is outside the docs/ directory (e.g. pyproject.toml in the repo root).
# ---------------------------------------------------------------------------
DOCS_BROKEN_LINKS: dict[str, str] = {}
SHORTNAME_MAP: dict[str, str] = {
# Short method refs that autorefs can't resolve without scope → full path
"setup": "[setup][syncmaster.server.providers.auth.AuthProvider.setup]",
}
# ---------------------------------------------------------------------------
# Transformer
# ---------------------------------------------------------------------------
def _fix_refs(text: str) -> tuple[str, list[str]]:
"""Replace [label][] and [text][onetl.*] with explicit Markdown links."""
changes: list[str] = []
def replace_ref(m: re.Match) -> str:
label = m.group(1)
if label in REF_MAP:
link_text, path = REF_MAP[label]
changes.append(f"ref: [{label}][] → [{link_text}]({path})")
return f"[{link_text}]({path})"
return m.group(0)
def replace_text_ref(m: re.Match) -> str:
link_text, label = m.group(1), m.group(2)
if label in REF_MAP:
_, path = REF_MAP[label]
changes.append(f"ref: [{link_text}][{label}] → [{link_text}]({path})")
return f"[{link_text}]({path})"
return m.group(0)
def replace_label_ref(m: re.Match) -> str:
link_text, label = m.group(1), m.group(2)
if label in LABEL_MAP:
url = LABEL_MAP[label]
changes.append(f"label: [{link_text}][{label}] → [{link_text}]({url})")
return f"[{link_text}]({url})"
return m.group(0)
def replace_python_path(m: re.Match) -> str:
link_text, python_path = m.group(1), m.group(2)
if python_path in PLAIN_REF_MAP:
name = PLAIN_REF_MAP[python_path]
changes.append(f"pyref: [{link_text}][{python_path}] → {name}")
return name
if python_path in PYTHON_PATH_MAP:
url = PYTHON_PATH_MAP[python_path]
changes.append(f"pyref: [{link_text}][{python_path}] → [{link_text}]({url})")
return f"[{link_text}]({url})"
return m.group(0)
def replace_python_path_self(m: re.Match) -> str:
"""Handle [onetl.module.ClassName][] where the path is also the display text."""
python_path = m.group(1)
if python_path in CODE_PATH_MAP:
name = CODE_PATH_MAP[python_path]
changes.append(f"pyref: [{python_path}][] → {name}")
return name
if python_path in ANCHOR_REF_MAP:
link_text, anchor = ANCHOR_REF_MAP[python_path]
changes.append(f"pyref: [{python_path}][] → [{link_text}][{anchor}]")
return f"[{link_text}][{anchor}]"
if python_path in PYTHON_PATH_MAP:
url = PYTHON_PATH_MAP[python_path]
# Use the last component (ClassName) as display text
link_text = python_path.rsplit(".", 1)[-1]
changes.append(f"pyref: [{python_path}][] → [{link_text}]({url})")
return f"[{link_text}]({url})"
return m.group(0)
def replace_shortname(m: re.Match) -> str:
name = m.group(1)
if name in SHORTNAME_MAP:
changes.append(f"shortname: [{name}][] → {SHORTNAME_MAP[name]}")
return SHORTNAME_MAP[name]
return m.group(0)
def fix_numpy_type(m: re.Match) -> str:
"""Remove RST-style backticks from NumPy parameter type fields."""
prefix, type_str = m.group(1), m.group(2)
# `T` or `T2` → T | T2 (both sides backticked)
fixed = re.sub(r"`([^`]+)`\s+or\s+`([^`]+)`", r"\1 | \2", type_str)
# remaining `T` → T (covers single types and default values)
fixed = re.sub(r"`([^`]+)`", r"\1", fixed)
# T or T2 → T | T2 (residual after backtick removal, or pre-existing)
fixed = re.sub(r"(\S)\s+or\s+(\S)", r"\1 | \2", fixed)
# list/Iterable of X → list[X] / Iterable[X]
fixed = re.sub(
r"\b(list|List|Iterable) of ([^\s,]+)", lambda mm: f"{mm.group(1).lower()}[{mm.group(2)}]", fixed
)
# typing.X → X (griffe cannot parse fully-qualified typing module refs)
fixed = re.sub(r"\btyping\.(\w+)", r"\1", fixed)
if fixed != type_str:
changes.append(f"type: {type_str!r} → {fixed!r}")
return prefix + fixed
return m.group(0)
def fix_see_list(m: re.Match) -> str:
"""Ensure blank line between 'See:' and the following bullet list.
Handles two cases:
1. Items over-indented by 4 spaces (RST style) → normalize to See: indent.
2. No blank line before items → add one.
"""
see_indent = m.group(1) # indentation of 'See:'
item_indent = m.group(2) # indentation of first '*'
rest = m.group(3) # everything from '*' to end of list block
# Normalize item indent to match See: indent
if len(item_indent) > len(see_indent):
# Re-indent all list items to See: indent level
def reindent(mm: re.Match) -> str:
return see_indent + "* " + mm.group(1)
rest = re.sub(r"[ \t]+\* (.+)", reindent, rest)
item_indent = see_indent
changes.append("see-list: added blank line before list after 'See:'")
return f"{see_indent}See:\n\n{item_indent}* {rest}"
def fix_label_list(m: re.Match) -> str:
"""Ensure blank line between any indented label ending with ':' and a bullet list."""
changes.append(f"label-list: added blank line after '{m.group(1).strip()}'")
return m.group(1) + "\n\n" + m.group(2)
def fix_numpy_params(txt: str) -> str:
"""Remove extra indentation from numpy-style Parameters/Returns/etc. sections.
Griffe expects:
Parameters
----------
param : type
description
But some docstrings have 4 extra spaces before each param and its description:
Parameters
----------
param : type
description
This function normalizes them to the expected format.
"""
lines = txt.split("\n")
out: list[str] = []
i = 0
while i < len(lines):
line = lines[i]
# Look for section header + underline
header_match = re.match(r"^( *)\S", line)
if header_match and i + 1 < len(lines):
underline = lines[i + 1]
if re.match(r"^ *-{3,}\s*$", underline) and line.rstrip() and not line.lstrip().startswith("-"):
base_indent = len(header_match.group(1))
expected_param_indent = base_indent
# Scan ahead to find actual param indent
j = i + 2
while j < len(lines) and not lines[j].strip():
j += 1
if j < len(lines):
first_param = lines[j]
actual_indent = len(first_param) - len(first_param.lstrip())
extra = actual_indent - expected_param_indent
if extra > 0 and first_param.strip():
# Collect section block and de-indent by extra spaces
out.append(line)
out.append(underline)
i += 2
fixed_any = False
while i < len(lines):
bl = lines[i]
bl_stripped = bl.lstrip()
bl_indent = len(bl) - len(bl_stripped) if bl_stripped else 0
# Section ends when a non-blank line is at base_indent level (new section or end)
if bl_stripped and bl_indent <= base_indent and not re.match(r"-{3,}", bl_stripped):
break
if bl_stripped and bl[:extra] == " " * extra:
out.append(bl[extra:])
fixed_any = True
else:
out.append(bl)
i += 1
if fixed_any:
changes.append(f"numpy-params: de-indented {extra}sp in section '{line.strip()}'")
continue
out.append(line)
i += 1
return "\n".join(out)
def fix_over_indented_lists(txt: str) -> str:
"""De-indent bullet list blocks that are over-indented relative to their label.
Targets blocks like:
Label:\n\n * item (items 4+ spaces deeper than label)
and normalizes item indent to match label indent.
Handles multi-paragraph list items (continues past blank lines within the block).
Terminates the block when a non-blank line at the SAME indent as items is not
a list item (e.g. an admonition or new paragraph at the same level).
Idempotent: no-op if item indent already equals label indent.
"""
lines = txt.split("\n")
out: list[str] = []
i = 0
while i < len(lines):
line = lines[i]
s = line.rstrip()
label_indent = len(s) - len(s.lstrip()) if s.strip() else 0
# Check: indented line ending with ':', followed by a blank line
if s and label_indent > 0 and s[-1] == ":" and i + 1 < len(lines) and not lines[i + 1].strip():
# Skip the blank line(s) to find the list
j = i + 1
while j < len(lines) and not lines[j].strip():
j += 1
if j < len(lines):
nxt = lines[j]
nxt_s = nxt.lstrip()
nxt_indent = len(nxt) - len(nxt_s)
extra = nxt_indent - label_indent
# Only fix when items are strictly deeper than label
if extra > 0 and nxt_s and nxt_s[0] in "*-+":
# Collect block continuing past blank lines.
# Terminate when a non-blank line satisfies:
# - indent < nxt_indent (de-indented past items), OR
# - indent == nxt_indent AND not a list item (new sibling block)
block: list[str] = []
k = j
while k < len(lines):
bl = lines[k]
if not bl.strip():
# Blank line: look ahead to decide if block continues
m = k + 1
while m < len(lines) and not lines[m].strip():
m += 1
if m >= len(lines):
break # end of text
nb = lines[m]
nb_s = nb.lstrip()
nb_indent = len(nb) - len(nb_s)
if nb_indent < nxt_indent:
break # de-indented: end of block
if nb_indent == nxt_indent and not (nb_s and nb_s[0] in "*-+" and nb_s[1:2] == " "):
break # same-level non-list content: end of block
block.append(bl)
k += 1
continue
bi = len(bl) - len(bl.lstrip())
if bi < nxt_indent:
break
block.append(bl)
k += 1
# De-indent each block line by `extra` spaces
new_block = [bl[extra:] if bl[:extra] == " " * extra else bl for bl in block]
changes.append(f"label-list: de-indented {extra}sp after '{s.strip()}'")
out.append(line)
# Preserve blank lines between label and block
out.extend(lines[i + 1 : j])
out.extend(new_block)
i = k
continue
out.append(line)
i += 1
return "\n".join(out)
new_text = re.sub(r"\[([a-z][a-z0-9]*(?:-[a-z0-9]+)*)\]\[\]", replace_ref, text)
new_text = re.sub(r"\[([^\]]+)\]\[([a-z][a-z0-9]*(?:-[a-z0-9]+)*)\]", replace_text_ref, new_text)
new_text = re.sub(r"\[([^\]]+)\]\[([A-Z][A-Za-z0-9]*)\]", replace_label_ref, new_text)
new_text = re.sub(r"\[([^\]]+)\]\[(syncmaster\.[a-zA-Z0-9_.]+)\]", replace_python_path, new_text)
new_text = re.sub(r"\[(syncmaster\.[a-zA-Z0-9_.]+)\]\[\]", replace_python_path_self, new_text)
new_text = re.sub(r"\[([A-Za-z][\w.]*)\]\[\]", replace_shortname, new_text)
# Global: List of X → list[X] (Returns sections and body text)
new_text = re.sub(r"\bList of ([^\s,]+)", r"list[\1]", new_text)
new_text = re.sub(r"^( {1,}\w+ : )(.+)$", fix_numpy_type, new_text, flags=re.MULTILINE)
# Fix See: blocks: ensure blank line before bullet list, normalize over-indented items.
# Only fires when there is NO blank line between See: and the first * (i.e. \n followed
# immediately by optional spaces + *), leaving already-correct blocks untouched.
new_text = re.sub(
r"^( *)See:\n(?!\n)( *)\* (.+(?:\n(?!\n)[ \t]*\* .+)*)",
fix_see_list,
new_text,
flags=re.MULTILINE,
)
# Fix any indented label ending with ':' that is immediately followed by a bullet list
# (no blank line between them). Applied after fix_see_list so 'See:' blocks are already
# handled and won't be double-processed.
new_text = re.sub(
r"^( +\w[^\n]*:)\n(?!\n)( +[*-] )",
fix_label_list,
new_text,
flags=re.MULTILINE,
)
# De-indent over-indented list blocks (items deeper than label → code block in Python-Markdown)
new_text = fix_over_indented_lists(new_text)
# Fix over-indented numpy Parameters/Returns/etc. sections (griffe cannot parse them)
new_text = fix_numpy_params(new_text)
return new_text, changes
# ---------------------------------------------------------------------------
# File processor (same structure as rst2md.py)
# ---------------------------------------------------------------------------
_TRIPLE_QUOTED = re.compile(
r'([rRbBuUfF]?"""(?:[^"\\]|\\.|"{1,2}(?!"))*"""|'
r"[rRbBuUfF]?'''(?:[^'\\]|\\.|'{1,2}(?!'))*''')",
re.DOTALL,
)
_NOQA_RE = re.compile(r"\s*#\s*noqa[^\n]*$")
def find_docstrings(source: str) -> list[tuple[int, int]]:
# Characters that indicate we're mid-expression (function call, list, etc.)
_expr_chars = frozenset("(,[{+-*/%&|^~\\")
results = []
for m in _TRIPLE_QUOTED.finditer(source):
preceding = _NOQA_RE.sub("", source[: m.start()].rstrip()).rstrip()
# Standard positions: module start, after 'def foo():', 'class Foo:', etc.
if preceding.endswith(":") or not preceding or preceding[-1] in ("\n", "#"):
results.append((m.start(), m.end()))
continue
# Field docstring: """ starts at the beginning of an indented line
# (only whitespace before it on the current line) and we're not mid-expression.
line_start = source.rfind("\n", 0, m.start()) + 1
if source[line_start : m.start()].strip() == "" and preceding[-1] not in _expr_chars:
results.append((m.start(), m.end()))
return results
def process_file(path: Path, *, dry_run: bool) -> bool:
try:
source = path.read_text(encoding="utf-8")
except Exception as e: # noqa: BLE001
print(f" ERROR reading {path}: {e}", file=sys.stderr)
return False
docstrings = find_docstrings(source)
if not docstrings:
return False
offset = 0
new_source = source
file_changes: list[str] = []
for start, end in docstrings:
raw = source[start:end]
quote = '"""' if raw.startswith('"""') else "'''"
body = raw[3:-3]
converted, changes = _fix_refs(body)
if not changes:
continue
new_raw = quote + converted + quote
adj_start = start + offset
adj_end = end + offset
new_source = new_source[:adj_start] + new_raw + new_source[adj_end:]
offset += len(new_raw) - len(raw)
file_changes.extend(changes)
if not file_changes:
return False
if dry_run:
print(f"\n {path}")
seen = set()
for c in file_changes:
if c not in seen:
print(f" · {c}")
seen.add(c)
else:
path.write_text(new_source, encoding="utf-8")
return True
# ---------------------------------------------------------------------------
# Docs .md file processor
# ---------------------------------------------------------------------------
def process_md_file(path: Path, *, dry_run: bool) -> bool:
try:
source = path.read_text(encoding="utf-8")
except Exception as e: # noqa: BLE001
print(f" ERROR reading {path}: {e}", file=sys.stderr)
return False
new_source = source
file_changes: list[str] = []
for old, new in DOCS_BROKEN_LINKS.items():
if old in new_source:
new_source = new_source.replace(old, new)
file_changes.append(f"broken-link: {old!r} → {new!r}")
if not file_changes:
return False
if dry_run:
print(f"\n {path}")
for c in file_changes:
print(f" · {c}")
else:
path.write_text(new_source, encoding="utf-8")
return True
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(
description="Fix doc-anchor cross-references in converted docstrings.",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--dry-run", action="store_true", help="Show changes without writing files")
parser.add_argument("--path", default="syncmaster/", help="Directory to process (default: syncmaster/)")
parser.add_argument("--file", help="Process a single file")
parser.add_argument("--docs-path", help="Also fix broken links in docs .md files under PATH")
args = parser.parse_args()
mode = "dry-run" if args.dry_run else "write"
files = [Path(args.file)] if args.file else sorted(Path(args.path).rglob("*.py"))
print(f"Processing {len(files)} file(s) [{mode}]...")
changed = 0
for f in files:
if process_file(f, dry_run=args.dry_run):
changed += 1
if not args.dry_run:
print(f" ✓ {f}")
label = "would be changed" if args.dry_run else "changed"
print(f"\nDone: {changed}/{len(files)} files {label}.")
if args.docs_path:
md_files = sorted(Path(args.docs_path).rglob("*.md"))
print(f"\nProcessing {len(md_files)} docs .md file(s) [{mode}]...")
md_changed = 0
for f in md_files:
if process_md_file(f, dry_run=args.dry_run):
md_changed += 1
if not args.dry_run:
print(f" ✓ {f}")
print(f"Done: {md_changed}/{len(md_files)} docs files {label}.")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
24.6 kB
·
Xet hash:
f79b5c8202aed589acc520693a28cef3fb861c4e15e6b2caf54c29d561b3d7bb

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.