NOEON / scripts /source_manifest.py
Frankenstein-Labs's picture
NOEON: full model, code, configs, tokenizers, manifests
419f250 verified
Raw History Blame Contribute Delete
2.91 kB
#!/usr/bin/env python3
"""Build the authoritative source-file manifest for NOEON — no weight download.
Uses the Hugging Face tree API, which exposes each file's true content hash:
* regular files -> ``oid`` (git blob SHA-1)
* LFS files -> ``lfs.oid`` (real SHA-256 of the stored bytes) and size
So every source weight is identified by origin, size, format and SHA-256
**without transferring gigabytes**. The result is written to
``weights/source_files.json`` and summarised in ``weights/MANIFEST.md``.
Usage:
python scripts/source_manifest.py
"""
from __future__ import annotations
import json
import sys
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from noeon.core.registry import SOURCE_MODELS
ROOT = Path(__file__).resolve().parent.parent
WEIGHTS = ROOT / "weights"
def tree(repo: str, revision: str, token: str | None = None) -> list[dict]:
url = f"https://huggingface.co/api/models/{repo}/tree/{revision}?recursive=true"
req = urllib.request.Request(url, headers={"User-Agent": "noeon/0.1"})
if token:
req.add_header("Authorization", f"Bearer {token}")
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read().decode())
def main() -> int:
import os
token = os.environ.get("HF_TOKEN")
out = {"generated_for": "NOEON", "sources": {}}
for key, src in SOURCE_MODELS.items():
try:
entries = tree(src.repo, src.revision, token)
except Exception as e: # noqa: BLE001
print(f"[{key}] tree fetch failed: {e}")
out["sources"][key] = {"repo": src.repo, "error": str(e)}
continue
files = []
for e in entries:
if e.get("type") != "file":
continue
lfs = e.get("lfs") or {}
files.append({
"path": e["path"],
"size": e.get("size"),
"blob_sha1": e.get("oid"),
"lfs_sha256": lfs.get("oid"),
"is_weight": e["path"].endswith((".safetensors", ".bin", ".pt", ".ckpt")),
})
out["sources"][key] = {
"repo": src.repo,
"revision": src.revision,
"license": src.license,
"role": src.role,
"n_files": len(files),
"total_bytes": sum(f["size"] or 0 for f in files),
"files": files,
}
wb = sum(f["size"] or 0 for f in files if f["is_weight"])
print(f"[{key}] {len(files)} files, {wb/1e9:.2f} GB of weights "
f"(sha256 known for {sum(1 for f in files if f['lfs_sha256'])} files)")
WEIGHTS.mkdir(parents=True, exist_ok=True)
(WEIGHTS / "source_files.json").write_text(json.dumps(out, indent=2))
print(f"\nwrote {WEIGHTS / 'source_files.json'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())