Download scripts/source_manifest.py from Frankenstein-Labs/NOEON: direct link, hf CLI and curl.
- Browser
- Download file 2.91 kB
-
https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/source_manifest.py
- Command line
-
hf download hf://Frankenstein-Labs/NOEON/scripts/source_manifest.py
-
curl -L -o source_manifest.py https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/source_manifest.py
2.91 kB
| #!/usr/bin/env python3 | |
| """Build the authoritative source-file manifest for NOEON — no weight download. | |
| Uses the Hugging Face tree API, which exposes each file's true content hash: | |
| * regular files -> ``oid`` (git blob SHA-1) | |
| * LFS files -> ``lfs.oid`` (real SHA-256 of the stored bytes) and size | |
| So every source weight is identified by origin, size, format and SHA-256 | |
| **without transferring gigabytes**. The result is written to | |
| ``weights/source_files.json`` and summarised in ``weights/MANIFEST.md``. | |
| Usage: | |
| python scripts/source_manifest.py | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| import urllib.request | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from noeon.core.registry import SOURCE_MODELS | |
| ROOT = Path(__file__).resolve().parent.parent | |
| WEIGHTS = ROOT / "weights" | |
| def tree(repo: str, revision: str, token: str | None = None) -> list[dict]: | |
| url = f"https://huggingface.co/api/models/{repo}/tree/{revision}?recursive=true" | |
| req = urllib.request.Request(url, headers={"User-Agent": "noeon/0.1"}) | |
| if token: | |
| req.add_header("Authorization", f"Bearer {token}") | |
| with urllib.request.urlopen(req, timeout=120) as r: | |
| return json.loads(r.read().decode()) | |
| def main() -> int: | |
| import os | |
| token = os.environ.get("HF_TOKEN") | |
| out = {"generated_for": "NOEON", "sources": {}} | |
| for key, src in SOURCE_MODELS.items(): | |
| try: | |
| entries = tree(src.repo, src.revision, token) | |
| except Exception as e: # noqa: BLE001 | |
| print(f"[{key}] tree fetch failed: {e}") | |
| out["sources"][key] = {"repo": src.repo, "error": str(e)} | |
| continue | |
| files = [] | |
| for e in entries: | |
| if e.get("type") != "file": | |
| continue | |
| lfs = e.get("lfs") or {} | |
| files.append({ | |
| "path": e["path"], | |
| "size": e.get("size"), | |
| "blob_sha1": e.get("oid"), | |
| "lfs_sha256": lfs.get("oid"), | |
| "is_weight": e["path"].endswith((".safetensors", ".bin", ".pt", ".ckpt")), | |
| }) | |
| out["sources"][key] = { | |
| "repo": src.repo, | |
| "revision": src.revision, | |
| "license": src.license, | |
| "role": src.role, | |
| "n_files": len(files), | |
| "total_bytes": sum(f["size"] or 0 for f in files), | |
| "files": files, | |
| } | |
| wb = sum(f["size"] or 0 for f in files if f["is_weight"]) | |
| print(f"[{key}] {len(files)} files, {wb/1e9:.2f} GB of weights " | |
| f"(sha256 known for {sum(1 for f in files if f['lfs_sha256'])} files)") | |
| WEIGHTS.mkdir(parents=True, exist_ok=True) | |
| (WEIGHTS / "source_files.json").write_text(json.dumps(out, indent=2)) | |
| print(f"\nwrote {WEIGHTS / 'source_files.json'}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |