| """Audit and export an explicit public source set without uploading anything. |
| |
| Git ignore rules protect ordinary staging; this separate allowlist also protects |
| folder-based uploads. Pattern scanning is a heuristic, not a rights review or a |
| complete secret detector. Findings report paths and categories, never matches. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import json |
| import re |
| import subprocess |
| import tempfile |
| from pathlib import Path, PurePosixPath |
|
|
| DENIED_PARTS = { |
| ".git", ".git_backup", ".venv", "node_modules", "__pycache__", "data", |
| "artifacts", "reports", "research", "private", "restricted", "models", |
| "weights", "checkpoints", "releases", ".aws", ".ssh", ".cache", |
| } |
| TEXT_SUFFIXES = {".py", ".ts", ".mjs", ".md", ".json", ".toml", ".yml", ".cff", ".html", ".css"} |
| TEXT_NAMES = {".gitignore", ".gitattributes", "LICENSE", "NOTICE"} |
| PATTERNS = { |
| "private-key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), |
| "provider-token": re.compile( |
| r"\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}" |
| r"|hf_[A-Za-z0-9]{25,}|sk-(?:proj-)?[A-Za-z0-9_-]{30,})\b" |
| ), |
| "aws-access-key": re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), |
| "personal-home-path": re.compile(r"/(?:Users|home)/[A-Za-z0-9_.-]+/"), |
| "private-writing-session": re.compile( |
| r"localhost[:]8766|/frames/[0-9a-f-]{36}|proj_[0-9a-f]{12}" |
| ), |
| "credential-url": re.compile(r"https?://[^\s/:@]+:[^\s/@]+@"), |
| } |
|
|
|
|
| def git_files(root: Path) -> set[str]: |
| result = subprocess.run( |
| ["git", "-c", "core.quotepath=false", "ls-files", "-z", "--cached", "--others", |
| "--exclude-standard"], cwd=root, check=True, capture_output=True, |
| ) |
| return {p for p in result.stdout.decode().split("\0") if p} |
|
|
|
|
| def inspect_file(root: Path, name: str, reviewed_binary: str | None = None) -> bytes: |
| relative = PurePosixPath(name) |
| if (relative.is_absolute() or ".." in relative.parts or "\\" in name |
| or relative.as_posix() != name or not relative.parts |
| or DENIED_PARTS.intersection(relative.parts) |
| or any(p.startswith(".env") for p in relative.parts)): |
| raise ValueError(f"disallowed path: {name}") |
| source = root / name |
| if any((root / Path(*relative.parts[:i])).is_symlink() |
| for i in range(1, len(relative.parts) + 1)): |
| raise ValueError(f"symlink is not public source: {name}") |
| if not source.is_file() or not source.resolve().is_relative_to(root.resolve()): |
| raise ValueError(f"missing or external source: {name}") |
| if source.stat().st_size > 20 * 1024 * 1024: |
| raise ValueError(f"oversized source: {name}") |
| content = source.read_bytes() |
| binary_location = ( |
| relative.suffix == ".pdf" and relative.parts[:2] == ("docs", "publications") |
| ) or (relative.suffix == ".png" and relative.parts[:2] == ("docs", "assets")) |
| if binary_location: |
| signature = b"%PDF-" if relative.suffix == ".pdf" else b"\x89PNG\r\n\x1a\n" |
| if (not content.startswith(signature) or not reviewed_binary |
| or hashlib.sha256(content).hexdigest() != reviewed_binary): |
| raise ValueError(f"binary needs an exact reviewed digest: {name}") |
| return content |
| if relative.suffix not in TEXT_SUFFIXES and relative.name not in TEXT_NAMES: |
| raise ValueError(f"unreviewed file type: {name}") |
| try: |
| text = content.decode("utf-8") |
| except UnicodeDecodeError as exc: |
| raise ValueError(f"non-text source: {name}") from exc |
| if "\0" in text: |
| raise ValueError(f"binary content in text file: {name}") |
| for category, pattern in PATTERNS.items(): |
| if pattern.search(text): |
| raise ValueError(f"{category}: {name}") |
| return content |
|
|
|
|
| def audit(root: Path) -> tuple[dict, dict[str, bytes]]: |
| spec = json.loads((root / "configs/public-release.json").read_text(encoding="utf-8")) |
| names = spec["files"] |
| if (not isinstance(names, list) or not names |
| or not all(isinstance(n, str) for n in names) or len(set(names)) != len(names)): |
| raise ValueError("release files must be a nonempty unique path list") |
| candidates = git_files(root) |
| if candidates != set(names): |
| raise ValueError(json.dumps({"unexpected_public_paths": sorted(candidates - set(names)), |
| "missing_or_ignored_paths": sorted(set(names) - candidates)})) |
| content = {name: inspect_file(root, name, spec.get("reviewed_binaries", {}).get(name)) |
| for name in sorted(names)} |
| return spec, content |
|
|
|
|
| def prepare(root: Path, output: Path | None = None, target: str = "github") -> dict: |
| spec, content = audit(root) |
| if target not in {"github", "huggingface"}: |
| raise ValueError("unknown release target") |
| if target == "huggingface": |
| content["README.md"] = content["docs/HUGGING_FACE.md"].replace(b"](../", b"](") |
| manifest = { |
| "schema": "vons.public-source/v1", "target": target, |
| "status": "prepared_local_only", "license_status": spec["license_status"], |
| "weights_included": False, "datasets_included": False, |
| "files": [{"path": name, "bytes": len(value), |
| "sha256": hashlib.sha256(value).hexdigest()} |
| for name, value in sorted(content.items())], |
| } |
| if output is not None: |
| output = output.resolve() |
| if output.exists(): |
| raise ValueError("refusing to overwrite a release directory") |
| if output == root.resolve() or output in root.resolve().parents: |
| raise ValueError("output must not replace the source tree") |
| output.parent.mkdir(parents=True, exist_ok=True) |
| with tempfile.TemporaryDirectory(dir=output.parent) as temporary: |
| stage = Path(temporary) / "release" |
| stage.mkdir() |
| for name, value in content.items(): |
| destination = stage / name |
| destination.parent.mkdir(parents=True, exist_ok=True) |
| destination.write_bytes(value) |
| (stage / "RELEASE_MANIFEST.json").write_text( |
| json.dumps(manifest, indent=2) + "\n", encoding="utf-8") |
| stage.rename(output) |
| return manifest |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) |
| parser.add_argument("--target", choices=["github", "huggingface"], default="github") |
| parser.add_argument("--output", type=Path, help="Create a new local export directory") |
| args = parser.parse_args() |
| try: |
| result = prepare(args.root.resolve(), args.output, args.target) |
| except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as exc: |
| parser.exit(1, f"Public release audit failed: {exc}\n") |
| print(json.dumps({"status": result["status"], "files": len(result["files"]), |
| "bytes": sum(f["bytes"] for f in result["files"]), |
| "license_status": result["license_status"], "target": args.target}, indent=2)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
