"""Build an offline static tutorial and a source-only release archive.""" import hashlib import html import json from pathlib import Path import re import sys import zipfile import markdown from markdown.extensions.toc import slugify_unicode ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from image_cleaner import __version__ REPO = "HanWangTJU/NMM-data-preprocess" DOCS = {"QUICKSTART": "逐步教程", "USAGE": "完整手册", "PARAMETERS": "参数参考", "DEPLOYMENT": "部署指南", "ARCHITECTURE": "代码说明"} SOURCE_DIRS = ("image_cleaner", "configs", "docs", "examples", "tests", "tools", "web", "docker") ROOT_FILES = ("README.md", "pyproject.toml", ".gitignore", ".dockerignore") ALLOWED_SUFFIXES = {".py", ".json", ".md", ".html", ".css", ".js"} def source_files(): paths = [ROOT/name for name in ROOT_FILES] for directory in SOURCE_DIRS: for path in (ROOT/directory).rglob("*"): if path.is_file() and (path.suffix in ALLOWED_SUFFIXES or path.name == "Dockerfile"): if "__pycache__" not in path.parts: paths.append(path) for path in paths: if path.is_symlink() or not path.resolve().is_relative_to(ROOT): raise ValueError("Publish source must remain inside the project") return sorted(set(paths)) def check_content(content): if re.search(rb"hf_[A-Za-z0-9]{20,}", content): raise ValueError("A credential-like Hugging Face token was detected; publication stopped") if re.search(rb"(?i)(?:password|passwd)\s*[:=]\s*['\"][^'\"]{4,}['\"]", content): raise ValueError("A password assignment was detected; publication stopped") def check_path(path): if path.is_symlink() or not path.resolve().is_relative_to(ROOT): raise ValueError("Publish path must remain inside the project") if path.suffix == ".zip": with zipfile.ZipFile(path) as archive: expected = {p.relative_to(ROOT).as_posix() for p in source_files()} if set(archive.namelist()) != expected: raise ValueError("Archive contents differ from source whitelist") for name in archive.namelist(): check_content(archive.read(name)) else: check_content(path.read_bytes()) def publish_files(): return source_files() + [ROOT/"index.html", ROOT/"assets/style.css", ROOT/"assets/app.js"] + [ ROOT/"manual"/(name + ".html") for name in DOCS ] + [ROOT/"downloads"/f"NMM-data-preprocess-{__version__}.zip"] def rewrite_links(body): def rewrite(match): target = html.unescape(match.group(1)) base, _, fragment = target.partition("#") if base.endswith(".md"): stem = Path(base).stem if stem in DOCS: return 'href="' + stem + ".html" + ("#"+fragment if fragment else "") + '"' return 'href="https://huggingface.co/spaces/' + REPO + '/blob/main/' + html.escape(base) + '"' return match.group(0) return re.sub(r'href="([^"]+)"', rewrite, body) def build(): for directory in ["manual", "assets", "downloads", "release"]: (ROOT/directory).mkdir(exist_ok=True) # ASCII entities prevent the static host's script injection from splitting # multibyte UTF-8 characters across its byte-buffer boundaries. homepage = (ROOT/"web/index.template.html").read_text(encoding="utf-8").replace("@@VERSION@@", __version__) (ROOT/"index.html").write_bytes(homepage.encode("ascii", errors="xmlcharrefreplace")) for filename in ["style.css", "app.js"]: (ROOT/"assets"/filename).write_bytes((ROOT/"web"/filename).read_bytes()) for name, title in DOCS.items(): renderer = markdown.Markdown(extensions=["toc", "tables", "fenced_code", "sane_lists"], extension_configs={"toc": {"toc_depth": "2-3", "slugify": slugify_unicode}}) body = rewrite_links(renderer.convert((ROOT/"docs"/(name+".md")).read_text(encoding="utf-8"))) nav = "".join(f'{label}' for key, label in DOCS.items()) page = ('' '' f'{title} · NMM-data-preprocess' '
' 'NMM / data-preprocess' f'
' f'
{body}
' f'') (ROOT/"manual"/(name+".html")).write_bytes(page.encode("ascii", errors="xmlcharrefreplace")) archive_path = ROOT/"downloads"/f"NMM-data-preprocess-{__version__}.zip" with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: for path in source_files(): check_path(path) # Fixed metadata makes the archive reproducible across local runs. entry = zipfile.ZipInfo(path.relative_to(ROOT).as_posix(), date_time=(2026, 1, 1, 0, 0, 0)) entry.compress_type = zipfile.ZIP_DEFLATED entry.external_attr = 0o644 << 16 archive.writestr(entry, path.read_bytes()) files = [] for path in publish_files(): check_path(path) data = path.read_bytes() files.append({"path": path.relative_to(ROOT).as_posix(), "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}) manifest = {"repo_id": REPO, "version": __version__, "files": files, "archive": archive_path.relative_to(ROOT).as_posix()} (ROOT/"release/publish-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+"\n", encoding="utf-8") print(json.dumps({"version": __version__, "publish_files": len(files), "publish_bytes": sum(item["bytes"] for item in files), "archive": manifest["archive"]})) if __name__ == "__main__": build()