Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /MTSWebServices_syncmaster /mddocs /scripts /fix_refs.py
| # SPDX-FileCopyrightText: 2025-present MTS PJSC | |
| # SPDX-License-Identifier: Apache-2.0 | |
| """ | |
| Fix unresolved doc-anchor references in converted docstrings. | |
| Replaces two kinds of unresolved cross-references: | |
| 1. [label][] — converted from RST :ref:, resolved via REF_MAP | |
| 2. [text][syncmaster.module.ClassName] — Python-path refs that autorefs cannot | |
| resolve (because show_root_heading: false suppresses anchor generation), | |
| resolved via PYTHON_PATH_MAP | |
| Both mapping tables are the single source of truth. Update them when doc pages | |
| are renamed, reorganised, or when new unresolved references appear. | |
| Usage: | |
| python fix_refs.py [--dry-run] [--path PATH] [--file FILE] [--docs-path PATH] | |
| Options: | |
| --dry-run Print changes without writing files. | |
| --path PATH Directory to process recursively (default: syncmaster/). | |
| --file FILE Process a single file instead of a directory. | |
| --docs-path PATH Also fix broken links in docs .md files under PATH. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import re | |
| import sys | |
| from pathlib import Path | |
| # --------------------------------------------------------------------------- | |
| # Mapping: RST :ref: label → (link text, absolute docs path) | |
| # | |
| # Paths are absolute from the docs root (as served by MkDocs). | |
| # Update this table when doc pages are renamed or reorganised. | |
| # --------------------------------------------------------------------------- | |
| REF_MAP: dict[str, tuple[str, str]] = { | |
| # All :ref: labels were converted manually during RST→MD migration. | |
| # Add entries here if new label-style references appear. | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Mapping: Python dotted path → absolute docs URL (with anchor) | |
| # | |
| # Used for [text][onetl.module.ClassName] cross-references that autorefs | |
| # cannot resolve because show_root_heading: false suppresses anchor generation | |
| # for documented root objects. | |
| # --------------------------------------------------------------------------- | |
| PYTHON_PATH_MAP: dict[str, str] = { | |
| # Add entries here for [text][syncmaster.*] refs that autorefs cannot resolve | |
| # because show_root_heading: false suppresses the anchor, mapping to explicit URLs. | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Mapping: short class name label → absolute docs URL with anchor | |
| # | |
| # Used for [text][ShortClassName] cross-references where the label is a plain | |
| # class name (not a full onetl.* path and not a kebab-case REF_MAP key). | |
| # --------------------------------------------------------------------------- | |
| LABEL_MAP: dict[str, str] = {} | |
| # --------------------------------------------------------------------------- | |
| # Mapping: Python dotted path → (display text, DBR anchor label) | |
| # | |
| # Used for [onetl.path][] cross-references where the target is documented | |
| # via a { #DBR-... } anchor (not a mkdocstrings ::: directive). | |
| # Output format: [display text][DBR-anchor] | |
| # --------------------------------------------------------------------------- | |
| ANCHOR_REF_MAP: dict[str, tuple[str, str]] = {} | |
| # --------------------------------------------------------------------------- | |
| # Mapping: Python dotted path → inline code name | |
| # | |
| # Used for [onetl.path][] cross-references where the target is NOT documented | |
| # and cannot be linked. Output: `ClassName` (inline code, no link). | |
| # --------------------------------------------------------------------------- | |
| CODE_PATH_MAP: dict[str, str] = { | |
| # Functions/classes not documented on any page → inline code (no link) | |
| "syncmaster.server.application_factory": "`application_factory`", | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Mapping: Python dotted path (wrong/legacy) → plain name | |
| # | |
| # Used for [text][onetl.path] where the path is incorrect or the target | |
| # should be rendered as a plain ExprName (let griffe/autorefs resolve it). | |
| # Output: just the name, no brackets. | |
| # --------------------------------------------------------------------------- | |
| PLAIN_REF_MAP: dict[str, str] = { | |
| # Wrong Python paths in docstrings → correct inline code | |
| # (target is not documented, so just render as code) | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Mapping: short [Name][] refs not in onetl.* namespace | |
| # | |
| # Used for [Name][] where Name is a simple identifier or dotted non-onetl path. | |
| # Values wrapped in backticks become inline code; plain strings become plain text. | |
| # --------------------------------------------------------------------------- | |
| # --------------------------------------------------------------------------- | |
| # Mapping: broken relative links in docs .md files → correct replacement | |
| # | |
| # Used for links that cannot be resolved by MkDocs because the target file | |
| # is outside the docs/ directory (e.g. pyproject.toml in the repo root). | |
| # --------------------------------------------------------------------------- | |
| DOCS_BROKEN_LINKS: dict[str, str] = {} | |
| SHORTNAME_MAP: dict[str, str] = { | |
| # Short method refs that autorefs can't resolve without scope → full path | |
| "setup": "[setup][syncmaster.server.providers.auth.AuthProvider.setup]", | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Transformer | |
| # --------------------------------------------------------------------------- | |
| def _fix_refs(text: str) -> tuple[str, list[str]]: | |
| """Replace [label][] and [text][onetl.*] with explicit Markdown links.""" | |
| changes: list[str] = [] | |
| def replace_ref(m: re.Match) -> str: | |
| label = m.group(1) | |
| if label in REF_MAP: | |
| link_text, path = REF_MAP[label] | |
| changes.append(f"ref: [{label}][] → [{link_text}]({path})") | |
| return f"[{link_text}]({path})" | |
| return m.group(0) | |
| def replace_text_ref(m: re.Match) -> str: | |
| link_text, label = m.group(1), m.group(2) | |
| if label in REF_MAP: | |
| _, path = REF_MAP[label] | |
| changes.append(f"ref: [{link_text}][{label}] → [{link_text}]({path})") | |
| return f"[{link_text}]({path})" | |
| return m.group(0) | |
| def replace_label_ref(m: re.Match) -> str: | |
| link_text, label = m.group(1), m.group(2) | |
| if label in LABEL_MAP: | |
| url = LABEL_MAP[label] | |
| changes.append(f"label: [{link_text}][{label}] → [{link_text}]({url})") | |
| return f"[{link_text}]({url})" | |
| return m.group(0) | |
| def replace_python_path(m: re.Match) -> str: | |
| link_text, python_path = m.group(1), m.group(2) | |
| if python_path in PLAIN_REF_MAP: | |
| name = PLAIN_REF_MAP[python_path] | |
| changes.append(f"pyref: [{link_text}][{python_path}] → {name}") | |
| return name | |
| if python_path in PYTHON_PATH_MAP: | |
| url = PYTHON_PATH_MAP[python_path] | |
| changes.append(f"pyref: [{link_text}][{python_path}] → [{link_text}]({url})") | |
| return f"[{link_text}]({url})" | |
| return m.group(0) | |
| def replace_python_path_self(m: re.Match) -> str: | |
| """Handle [onetl.module.ClassName][] where the path is also the display text.""" | |
| python_path = m.group(1) | |
| if python_path in CODE_PATH_MAP: | |
| name = CODE_PATH_MAP[python_path] | |
| changes.append(f"pyref: [{python_path}][] → {name}") | |
| return name | |
| if python_path in ANCHOR_REF_MAP: | |
| link_text, anchor = ANCHOR_REF_MAP[python_path] | |
| changes.append(f"pyref: [{python_path}][] → [{link_text}][{anchor}]") | |
| return f"[{link_text}][{anchor}]" | |
| if python_path in PYTHON_PATH_MAP: | |
| url = PYTHON_PATH_MAP[python_path] | |
| # Use the last component (ClassName) as display text | |
| link_text = python_path.rsplit(".", 1)[-1] | |
| changes.append(f"pyref: [{python_path}][] → [{link_text}]({url})") | |
| return f"[{link_text}]({url})" | |
| return m.group(0) | |
| def replace_shortname(m: re.Match) -> str: | |
| name = m.group(1) | |
| if name in SHORTNAME_MAP: | |
| changes.append(f"shortname: [{name}][] → {SHORTNAME_MAP[name]}") | |
| return SHORTNAME_MAP[name] | |
| return m.group(0) | |
| def fix_numpy_type(m: re.Match) -> str: | |
| """Remove RST-style backticks from NumPy parameter type fields.""" | |
| prefix, type_str = m.group(1), m.group(2) | |
| # `T` or `T2` → T | T2 (both sides backticked) | |
| fixed = re.sub(r"`([^`]+)`\s+or\s+`([^`]+)`", r"\1 | \2", type_str) | |
| # remaining `T` → T (covers single types and default values) | |
| fixed = re.sub(r"`([^`]+)`", r"\1", fixed) | |
| # T or T2 → T | T2 (residual after backtick removal, or pre-existing) | |
| fixed = re.sub(r"(\S)\s+or\s+(\S)", r"\1 | \2", fixed) | |
| # list/Iterable of X → list[X] / Iterable[X] | |
| fixed = re.sub( | |
| r"\b(list|List|Iterable) of ([^\s,]+)", lambda mm: f"{mm.group(1).lower()}[{mm.group(2)}]", fixed | |
| ) | |
| # typing.X → X (griffe cannot parse fully-qualified typing module refs) | |
| fixed = re.sub(r"\btyping\.(\w+)", r"\1", fixed) | |
| if fixed != type_str: | |
| changes.append(f"type: {type_str!r} → {fixed!r}") | |
| return prefix + fixed | |
| return m.group(0) | |
| def fix_see_list(m: re.Match) -> str: | |
| """Ensure blank line between 'See:' and the following bullet list. | |
| Handles two cases: | |
| 1. Items over-indented by 4 spaces (RST style) → normalize to See: indent. | |
| 2. No blank line before items → add one. | |
| """ | |
| see_indent = m.group(1) # indentation of 'See:' | |
| item_indent = m.group(2) # indentation of first '*' | |
| rest = m.group(3) # everything from '*' to end of list block | |
| # Normalize item indent to match See: indent | |
| if len(item_indent) > len(see_indent): | |
| # Re-indent all list items to See: indent level | |
| def reindent(mm: re.Match) -> str: | |
| return see_indent + "* " + mm.group(1) | |
| rest = re.sub(r"[ \t]+\* (.+)", reindent, rest) | |
| item_indent = see_indent | |
| changes.append("see-list: added blank line before list after 'See:'") | |
| return f"{see_indent}See:\n\n{item_indent}* {rest}" | |
| def fix_label_list(m: re.Match) -> str: | |
| """Ensure blank line between any indented label ending with ':' and a bullet list.""" | |
| changes.append(f"label-list: added blank line after '{m.group(1).strip()}'") | |
| return m.group(1) + "\n\n" + m.group(2) | |
| def fix_numpy_params(txt: str) -> str: | |
| """Remove extra indentation from numpy-style Parameters/Returns/etc. sections. | |
| Griffe expects: | |
| Parameters | |
| ---------- | |
| param : type | |
| description | |
| But some docstrings have 4 extra spaces before each param and its description: | |
| Parameters | |
| ---------- | |
| param : type | |
| description | |
| This function normalizes them to the expected format. | |
| """ | |
| lines = txt.split("\n") | |
| out: list[str] = [] | |
| i = 0 | |
| while i < len(lines): | |
| line = lines[i] | |
| # Look for section header + underline | |
| header_match = re.match(r"^( *)\S", line) | |
| if header_match and i + 1 < len(lines): | |
| underline = lines[i + 1] | |
| if re.match(r"^ *-{3,}\s*$", underline) and line.rstrip() and not line.lstrip().startswith("-"): | |
| base_indent = len(header_match.group(1)) | |
| expected_param_indent = base_indent | |
| # Scan ahead to find actual param indent | |
| j = i + 2 | |
| while j < len(lines) and not lines[j].strip(): | |
| j += 1 | |
| if j < len(lines): | |
| first_param = lines[j] | |
| actual_indent = len(first_param) - len(first_param.lstrip()) | |
| extra = actual_indent - expected_param_indent | |
| if extra > 0 and first_param.strip(): | |
| # Collect section block and de-indent by extra spaces | |
| out.append(line) | |
| out.append(underline) | |
| i += 2 | |
| fixed_any = False | |
| while i < len(lines): | |
| bl = lines[i] | |
| bl_stripped = bl.lstrip() | |
| bl_indent = len(bl) - len(bl_stripped) if bl_stripped else 0 | |
| # Section ends when a non-blank line is at base_indent level (new section or end) | |
| if bl_stripped and bl_indent <= base_indent and not re.match(r"-{3,}", bl_stripped): | |
| break | |
| if bl_stripped and bl[:extra] == " " * extra: | |
| out.append(bl[extra:]) | |
| fixed_any = True | |
| else: | |
| out.append(bl) | |
| i += 1 | |
| if fixed_any: | |
| changes.append(f"numpy-params: de-indented {extra}sp in section '{line.strip()}'") | |
| continue | |
| out.append(line) | |
| i += 1 | |
| return "\n".join(out) | |
| def fix_over_indented_lists(txt: str) -> str: | |
| """De-indent bullet list blocks that are over-indented relative to their label. | |
| Targets blocks like: | |
| Label:\n\n * item (items 4+ spaces deeper than label) | |
| and normalizes item indent to match label indent. | |
| Handles multi-paragraph list items (continues past blank lines within the block). | |
| Terminates the block when a non-blank line at the SAME indent as items is not | |
| a list item (e.g. an admonition or new paragraph at the same level). | |
| Idempotent: no-op if item indent already equals label indent. | |
| """ | |
| lines = txt.split("\n") | |
| out: list[str] = [] | |
| i = 0 | |
| while i < len(lines): | |
| line = lines[i] | |
| s = line.rstrip() | |
| label_indent = len(s) - len(s.lstrip()) if s.strip() else 0 | |
| # Check: indented line ending with ':', followed by a blank line | |
| if s and label_indent > 0 and s[-1] == ":" and i + 1 < len(lines) and not lines[i + 1].strip(): | |
| # Skip the blank line(s) to find the list | |
| j = i + 1 | |
| while j < len(lines) and not lines[j].strip(): | |
| j += 1 | |
| if j < len(lines): | |
| nxt = lines[j] | |
| nxt_s = nxt.lstrip() | |
| nxt_indent = len(nxt) - len(nxt_s) | |
| extra = nxt_indent - label_indent | |
| # Only fix when items are strictly deeper than label | |
| if extra > 0 and nxt_s and nxt_s[0] in "*-+": | |
| # Collect block continuing past blank lines. | |
| # Terminate when a non-blank line satisfies: | |
| # - indent < nxt_indent (de-indented past items), OR | |
| # - indent == nxt_indent AND not a list item (new sibling block) | |
| block: list[str] = [] | |
| k = j | |
| while k < len(lines): | |
| bl = lines[k] | |
| if not bl.strip(): | |
| # Blank line: look ahead to decide if block continues | |
| m = k + 1 | |
| while m < len(lines) and not lines[m].strip(): | |
| m += 1 | |
| if m >= len(lines): | |
| break # end of text | |
| nb = lines[m] | |
| nb_s = nb.lstrip() | |
| nb_indent = len(nb) - len(nb_s) | |
| if nb_indent < nxt_indent: | |
| break # de-indented: end of block | |
| if nb_indent == nxt_indent and not (nb_s and nb_s[0] in "*-+" and nb_s[1:2] == " "): | |
| break # same-level non-list content: end of block | |
| block.append(bl) | |
| k += 1 | |
| continue | |
| bi = len(bl) - len(bl.lstrip()) | |
| if bi < nxt_indent: | |
| break | |
| block.append(bl) | |
| k += 1 | |
| # De-indent each block line by `extra` spaces | |
| new_block = [bl[extra:] if bl[:extra] == " " * extra else bl for bl in block] | |
| changes.append(f"label-list: de-indented {extra}sp after '{s.strip()}'") | |
| out.append(line) | |
| # Preserve blank lines between label and block | |
| out.extend(lines[i + 1 : j]) | |
| out.extend(new_block) | |
| i = k | |
| continue | |
| out.append(line) | |
| i += 1 | |
| return "\n".join(out) | |
| new_text = re.sub(r"\[([a-z][a-z0-9]*(?:-[a-z0-9]+)*)\]\[\]", replace_ref, text) | |
| new_text = re.sub(r"\[([^\]]+)\]\[([a-z][a-z0-9]*(?:-[a-z0-9]+)*)\]", replace_text_ref, new_text) | |
| new_text = re.sub(r"\[([^\]]+)\]\[([A-Z][A-Za-z0-9]*)\]", replace_label_ref, new_text) | |
| new_text = re.sub(r"\[([^\]]+)\]\[(syncmaster\.[a-zA-Z0-9_.]+)\]", replace_python_path, new_text) | |
| new_text = re.sub(r"\[(syncmaster\.[a-zA-Z0-9_.]+)\]\[\]", replace_python_path_self, new_text) | |
| new_text = re.sub(r"\[([A-Za-z][\w.]*)\]\[\]", replace_shortname, new_text) | |
| # Global: List of X → list[X] (Returns sections and body text) | |
| new_text = re.sub(r"\bList of ([^\s,]+)", r"list[\1]", new_text) | |
| new_text = re.sub(r"^( {1,}\w+ : )(.+)$", fix_numpy_type, new_text, flags=re.MULTILINE) | |
| # Fix See: blocks: ensure blank line before bullet list, normalize over-indented items. | |
| # Only fires when there is NO blank line between See: and the first * (i.e. \n followed | |
| # immediately by optional spaces + *), leaving already-correct blocks untouched. | |
| new_text = re.sub( | |
| r"^( *)See:\n(?!\n)( *)\* (.+(?:\n(?!\n)[ \t]*\* .+)*)", | |
| fix_see_list, | |
| new_text, | |
| flags=re.MULTILINE, | |
| ) | |
| # Fix any indented label ending with ':' that is immediately followed by a bullet list | |
| # (no blank line between them). Applied after fix_see_list so 'See:' blocks are already | |
| # handled and won't be double-processed. | |
| new_text = re.sub( | |
| r"^( +\w[^\n]*:)\n(?!\n)( +[*-] )", | |
| fix_label_list, | |
| new_text, | |
| flags=re.MULTILINE, | |
| ) | |
| # De-indent over-indented list blocks (items deeper than label → code block in Python-Markdown) | |
| new_text = fix_over_indented_lists(new_text) | |
| # Fix over-indented numpy Parameters/Returns/etc. sections (griffe cannot parse them) | |
| new_text = fix_numpy_params(new_text) | |
| return new_text, changes | |
| # --------------------------------------------------------------------------- | |
| # File processor (same structure as rst2md.py) | |
| # --------------------------------------------------------------------------- | |
| _TRIPLE_QUOTED = re.compile( | |
| r'([rRbBuUfF]?"""(?:[^"\\]|\\.|"{1,2}(?!"))*"""|' | |
| r"[rRbBuUfF]?'''(?:[^'\\]|\\.|'{1,2}(?!'))*''')", | |
| re.DOTALL, | |
| ) | |
| _NOQA_RE = re.compile(r"\s*#\s*noqa[^\n]*$") | |
| def find_docstrings(source: str) -> list[tuple[int, int]]: | |
| # Characters that indicate we're mid-expression (function call, list, etc.) | |
| _expr_chars = frozenset("(,[{+-*/%&|^~\\") | |
| results = [] | |
| for m in _TRIPLE_QUOTED.finditer(source): | |
| preceding = _NOQA_RE.sub("", source[: m.start()].rstrip()).rstrip() | |
| # Standard positions: module start, after 'def foo():', 'class Foo:', etc. | |
| if preceding.endswith(":") or not preceding or preceding[-1] in ("\n", "#"): | |
| results.append((m.start(), m.end())) | |
| continue | |
| # Field docstring: """ starts at the beginning of an indented line | |
| # (only whitespace before it on the current line) and we're not mid-expression. | |
| line_start = source.rfind("\n", 0, m.start()) + 1 | |
| if source[line_start : m.start()].strip() == "" and preceding[-1] not in _expr_chars: | |
| results.append((m.start(), m.end())) | |
| return results | |
| def process_file(path: Path, *, dry_run: bool) -> bool: | |
| try: | |
| source = path.read_text(encoding="utf-8") | |
| except Exception as e: # noqa: BLE001 | |
| print(f" ERROR reading {path}: {e}", file=sys.stderr) | |
| return False | |
| docstrings = find_docstrings(source) | |
| if not docstrings: | |
| return False | |
| offset = 0 | |
| new_source = source | |
| file_changes: list[str] = [] | |
| for start, end in docstrings: | |
| raw = source[start:end] | |
| quote = '"""' if raw.startswith('"""') else "'''" | |
| body = raw[3:-3] | |
| converted, changes = _fix_refs(body) | |
| if not changes: | |
| continue | |
| new_raw = quote + converted + quote | |
| adj_start = start + offset | |
| adj_end = end + offset | |
| new_source = new_source[:adj_start] + new_raw + new_source[adj_end:] | |
| offset += len(new_raw) - len(raw) | |
| file_changes.extend(changes) | |
| if not file_changes: | |
| return False | |
| if dry_run: | |
| print(f"\n {path}") | |
| seen = set() | |
| for c in file_changes: | |
| if c not in seen: | |
| print(f" · {c}") | |
| seen.add(c) | |
| else: | |
| path.write_text(new_source, encoding="utf-8") | |
| return True | |
| # --------------------------------------------------------------------------- | |
| # Docs .md file processor | |
| # --------------------------------------------------------------------------- | |
| def process_md_file(path: Path, *, dry_run: bool) -> bool: | |
| try: | |
| source = path.read_text(encoding="utf-8") | |
| except Exception as e: # noqa: BLE001 | |
| print(f" ERROR reading {path}: {e}", file=sys.stderr) | |
| return False | |
| new_source = source | |
| file_changes: list[str] = [] | |
| for old, new in DOCS_BROKEN_LINKS.items(): | |
| if old in new_source: | |
| new_source = new_source.replace(old, new) | |
| file_changes.append(f"broken-link: {old!r} → {new!r}") | |
| if not file_changes: | |
| return False | |
| if dry_run: | |
| print(f"\n {path}") | |
| for c in file_changes: | |
| print(f" · {c}") | |
| else: | |
| path.write_text(new_source, encoding="utf-8") | |
| return True | |
| # --------------------------------------------------------------------------- | |
| # Entry point | |
| # --------------------------------------------------------------------------- | |
| def main() -> None: | |
| parser = argparse.ArgumentParser( | |
| description="Fix doc-anchor cross-references in converted docstrings.", | |
| formatter_class=argparse.RawDescriptionHelpFormatter, | |
| ) | |
| parser.add_argument("--dry-run", action="store_true", help="Show changes without writing files") | |
| parser.add_argument("--path", default="syncmaster/", help="Directory to process (default: syncmaster/)") | |
| parser.add_argument("--file", help="Process a single file") | |
| parser.add_argument("--docs-path", help="Also fix broken links in docs .md files under PATH") | |
| args = parser.parse_args() | |
| mode = "dry-run" if args.dry_run else "write" | |
| files = [Path(args.file)] if args.file else sorted(Path(args.path).rglob("*.py")) | |
| print(f"Processing {len(files)} file(s) [{mode}]...") | |
| changed = 0 | |
| for f in files: | |
| if process_file(f, dry_run=args.dry_run): | |
| changed += 1 | |
| if not args.dry_run: | |
| print(f" ✓ {f}") | |
| label = "would be changed" if args.dry_run else "changed" | |
| print(f"\nDone: {changed}/{len(files)} files {label}.") | |
| if args.docs_path: | |
| md_files = sorted(Path(args.docs_path).rglob("*.md")) | |
| print(f"\nProcessing {len(md_files)} docs .md file(s) [{mode}]...") | |
| md_changed = 0 | |
| for f in md_files: | |
| if process_md_file(f, dry_run=args.dry_run): | |
| md_changed += 1 | |
| if not args.dry_run: | |
| print(f" ✓ {f}") | |
| print(f"Done: {md_changed}/{len(md_files)} docs files {label}.") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 24.6 kB
- Xet hash:
- f79b5c8202aed589acc520693a28cef3fb861c4e15e6b2caf54c29d561b3d7bb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.