Buckets:

download
raw
9.64 kB
#!/usr/bin/env python3
"""Validate href/src targets in the built documentation site.
Run after ``zensical build`` (default ``site/``). Exits with code 1 if any
checked link is invalid.
By default only **local** links (under the site directory) are validated, so
CI does not depend on live docs or third-party HTTP behavior. Pass
``--external`` to also request every ``http(s)`` URL (slower; may need tuning).
Args:
argv: See ``--help``.
"""
from __future__ import annotations
import argparse
import re
import sys
import urllib.parse
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
import requests
from bs4 import BeautifulSoup
USER_AGENT = "codecarbon-docs-link-check/1.0 (+https://github.com/mlco2/codecarbon)"
REQUEST_TIMEOUT = 15.0
MAX_WORKERS = 12
IGNORE_EXTERNAL_PREFIXES: tuple[str, ...] = (
"mailto:",
"tel:",
"javascript:",
)
IGNORE_EXTERNAL_GLOBS: tuple[str, ...] = (
"https://fonts.googleapis.com/*",
"https://fonts.gstatic.com/*",
)
SKIP_LOCAL_HREFS: frozenset[str] = frozenset({".", "..", "./", "../"})
def _is_github_host(url: str) -> bool:
"""True if URL host is github.com or a subdomain (not e.g. evilgithub.com)."""
host = (urllib.parse.urlparse(url).hostname or "").lower()
return host == "github.com" or host.endswith(".github.com")
def _matches_glob(url: str, pattern: str) -> bool:
from fnmatch import fnmatch
return fnmatch(url, pattern)
def _local_href_skip_ok(raw_href: str) -> tuple[bool, str] | None:
"""Return (True, reason) if href needs no filesystem check; None to resolve path."""
if not raw_href or raw_href.startswith("#"):
return True, "fragment-only"
if raw_href in SKIP_LOCAL_HREFS:
return True, "skip-navigation-href"
parsed = urllib.parse.urlsplit(raw_href)
if parsed.scheme in ("http", "https", "mailto", "tel", "javascript", "data"):
if parsed.scheme in ("http", "https"):
return True, "external-delegated"
return True, "skipped-scheme"
if parsed.netloc:
return True, "other-scheme"
return None
def _resolved_candidate_under_site(
site_root: Path, source_html: Path, path_part: str
) -> tuple[Path | None, str]:
"""Resolve path to absolute Path under site_root, or (None, error)."""
if path_part.startswith("/"):
candidate = (site_root / path_part.lstrip("/")).resolve()
else:
candidate = (source_html.parent / path_part).resolve()
try:
candidate.relative_to(site_root.resolve())
except ValueError:
return None, f"escapes site root: {candidate}"
return candidate, ""
def _match_local_file(
site_root: Path, candidate: Path, fragment: str
) -> tuple[bool, str]:
"""Return (ok, detail) after checking candidate against built site layout."""
if candidate.is_dir():
index_file = candidate / "index.html"
if index_file.is_file():
ok = _fragment_ok(index_file, fragment)
return ok, str(index_file) + (f"#{fragment}" if fragment else "")
if candidate.name == "how-to" and candidate == site_root / "how-to":
return True, "skip-how-to-section-root-without-index"
return False, f"missing index in directory {candidate}"
if candidate.is_file():
ok = _fragment_ok(candidate, fragment)
return ok, str(candidate) + (f"#{fragment}" if fragment else "")
if candidate.suffix == ".md":
html_dir = candidate.with_suffix("")
idx = html_dir / "index.html"
if idx.is_file():
return _fragment_ok(idx, fragment), str(idx)
if html_dir.is_dir() and (html_dir / "index.html").is_file():
idx2 = html_dir / "index.html"
return _fragment_ok(idx2, fragment), str(idx2)
html_path = candidate.with_suffix(".html")
if html_path.is_file():
return _fragment_ok(html_path, fragment), str(candidate)
nested = candidate.parent / candidate.name / "index.html"
if candidate.parent.is_dir() and nested.is_file():
return _fragment_ok(nested, fragment), str(nested)
return False, f"not found: {candidate}"
def _local_target_exists(
site_root: Path, source_html: Path, raw_href: str
) -> tuple[bool, str]:
"""Return (ok, detail) for a same-origin path or relative URL."""
skip = _local_href_skip_ok(raw_href)
if skip is not None:
return skip
parsed = urllib.parse.urlsplit(raw_href)
path_part = parsed.path or "."
candidate, err = _resolved_candidate_under_site(site_root, source_html, path_part)
if candidate is None:
return False, err
return _match_local_file(site_root, candidate, parsed.fragment)
def _fragment_ok(html_file: Path, fragment: str) -> bool:
if not fragment:
return True
text = html_file.read_text(encoding="utf-8", errors="replace")
if re.search(rf'\b(?:id|name)=["\']{re.escape(fragment)}["\']', text):
return True
slug_pat = rf'\bid=["\']{re.escape(fragment)}["\']'
return re.search(slug_pat, text) is not None
def _check_external(session: requests.Session, url: str) -> tuple[bool, str]:
if url.startswith("http://localhost") or url.startswith("http://127.0.0.1"):
return True, "skip-localhost"
for pat in IGNORE_EXTERNAL_GLOBS:
if _matches_glob(url, pat):
return True, "ignored-glob"
try:
r = session.head(url, allow_redirects=True, timeout=REQUEST_TIMEOUT)
if r.status_code in (405, 501):
r = session.get(
url, allow_redirects=True, timeout=REQUEST_TIMEOUT, stream=True
)
r.close()
if r.status_code == 403 and _is_github_host(url):
return True, "skip-github-403"
if 200 <= r.status_code < 400:
return True, str(r.status_code)
return False, f"HTTP {r.status_code}"
except requests.RequestException as e:
return False, str(e)
def collect_checks(site_root: Path) -> list[tuple[Path, str, str, str]]:
"""Rows: (source_html, attr_name, url, kind) kind is local|external."""
rows: list[tuple[Path, str, str, str]] = []
for html_path in sorted(site_root.rglob("*.html")):
if "assets" in html_path.parts and "javascripts" in html_path.parts:
continue
soup = BeautifulSoup(
html_path.read_text(encoding="utf-8", errors="replace"), "html.parser"
)
for tag in soup.find_all("a", href=True):
href = tag["href"].strip()
if href.startswith(IGNORE_EXTERNAL_PREFIXES):
continue
if (
href.startswith("http://")
or href.startswith("https://")
or href.startswith("//")
):
rows.append((html_path, "href", href, "external"))
else:
rows.append((html_path, "href", href, "local"))
for tag in soup.find_all("img", src=True):
src = tag["src"].strip()
if src.startswith("data:"):
continue
if (
src.startswith("http://")
or src.startswith("https://")
or src.startswith("//")
):
rows.append((html_path, "src", src, "external"))
else:
rows.append((html_path, "src", src, "local"))
return rows
def main() -> int:
parser = argparse.ArgumentParser(
description="Check links in built docs under site/."
)
parser.add_argument(
"site_dir",
nargs="?",
default="site",
type=Path,
help="Built site directory (default: site)",
)
parser.add_argument(
"--external",
action="store_true",
help="Also validate http(s) URLs (slow; optional CI / manual audit).",
)
args = parser.parse_args()
site_root = args.site_dir.resolve()
if not site_root.is_dir():
print(f"error: site directory not found: {site_root}", file=sys.stderr)
return 1
rows = collect_checks(site_root)
failures: list[str] = []
for src_html, attr, url, kind in rows:
if kind == "local":
ok, detail = _local_target_exists(site_root, src_html, url)
if not ok:
rel = src_html.relative_to(site_root)
failures.append(f"{rel}: {attr}={url!r} -> {detail}")
if args.external:
external_urls = sorted({url for _, _, url, kind in rows if kind == "external"})
session = requests.Session()
session.headers["User-Agent"] = USER_AGENT
def check_one(url: str) -> tuple[str, bool, str]:
ok, msg = _check_external(session, url)
return url, ok, msg
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
futures = {pool.submit(check_one, u): u for u in external_urls}
for fut in as_completed(futures):
url, ok, msg = fut.result()
if not ok:
failures.append(f"external: {url!r} -> {msg}")
if failures:
print("Documentation link check failed:", file=sys.stderr)
for line in sorted(failures):
print(f" {line}", file=sys.stderr)
return 1
n_ext = len({u for _, _, u, k in rows if k == "external"})
if args.external:
ext_note = f", {n_ext} external URLs checked"
else:
ext_note = f", {n_ext} external URLs not checked (use --external)"
print(f"Documentation link check passed ({len(rows)} attributes{ext_note}).")
return 0
if __name__ == "__main__":
raise SystemExit(main())

Xet Storage Details

Size:
9.64 kB
·
Xet hash:
99ead44b2cffda0020fb4b04448e2f1f7d5508981fea0a664eba5f27be602131

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.