#!/usr/bin/env python3 """Build the authoritative source-file manifest for NOEON — no weight download. Uses the Hugging Face tree API, which exposes each file's true content hash: * regular files -> ``oid`` (git blob SHA-1) * LFS files -> ``lfs.oid`` (real SHA-256 of the stored bytes) and size So every source weight is identified by origin, size, format and SHA-256 **without transferring gigabytes**. The result is written to ``weights/source_files.json`` and summarised in ``weights/MANIFEST.md``. Usage: python scripts/source_manifest.py """ from __future__ import annotations import json import sys import urllib.request from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from noeon.core.registry import SOURCE_MODELS ROOT = Path(__file__).resolve().parent.parent WEIGHTS = ROOT / "weights" def tree(repo: str, revision: str, token: str | None = None) -> list[dict]: url = f"https://huggingface.co/api/models/{repo}/tree/{revision}?recursive=true" req = urllib.request.Request(url, headers={"User-Agent": "noeon/0.1"}) if token: req.add_header("Authorization", f"Bearer {token}") with urllib.request.urlopen(req, timeout=120) as r: return json.loads(r.read().decode()) def main() -> int: import os token = os.environ.get("HF_TOKEN") out = {"generated_for": "NOEON", "sources": {}} for key, src in SOURCE_MODELS.items(): try: entries = tree(src.repo, src.revision, token) except Exception as e: # noqa: BLE001 print(f"[{key}] tree fetch failed: {e}") out["sources"][key] = {"repo": src.repo, "error": str(e)} continue files = [] for e in entries: if e.get("type") != "file": continue lfs = e.get("lfs") or {} files.append({ "path": e["path"], "size": e.get("size"), "blob_sha1": e.get("oid"), "lfs_sha256": lfs.get("oid"), "is_weight": e["path"].endswith((".safetensors", ".bin", ".pt", ".ckpt")), }) out["sources"][key] = { "repo": src.repo, "revision": src.revision, "license": src.license, "role": src.role, "n_files": len(files), "total_bytes": sum(f["size"] or 0 for f in files), "files": files, } wb = sum(f["size"] or 0 for f in files if f["is_weight"]) print(f"[{key}] {len(files)} files, {wb/1e9:.2f} GB of weights " f"(sha256 known for {sum(1 for f in files if f['lfs_sha256'])} files)") WEIGHTS.mkdir(parents=True, exist_ok=True) (WEIGHTS / "source_files.json").write_text(json.dumps(out, indent=2)) print(f"\nwrote {WEIGHTS / 'source_files.json'}") return 0 if __name__ == "__main__": raise SystemExit(main())