#!/usr/bin/env python3 """Download the small, real files NOEON needs to *load* its sources. Weights are large (tens of GB) and are NOT downloaded here — they are described by SHA-256 in ``weights/source_files.json`` and can be fetched with ``huggingface-cli download``. This script fetches only the loading files: configs, tokenizer, preprocessor and the upstream modeling code, i.e. everything needed for ``from_pretrained`` other than the parameters themselves. It deliberately skips README/assets/examples/codec tarballs (not needed to load). Every downloaded file is verified against the SHA-256 recorded by the Hub tree API, so the copies are provably identical to the originals. Usage: python scripts/fetch_loading_files.py """ from __future__ import annotations import hashlib import json import os import sys import urllib.request from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from noeon.core.registry import SOURCE_MODELS ROOT = Path(__file__).resolve().parent.parent OUT = ROOT / "weights" / "source_loading_files" KEEP_SUFFIXES = ( ".json", ".jinja", ".txt", ".py", ".model", ".tiktoken", ) KEEP_NAMES = { "tokenizer_config.json", "preprocessor_config.json", "processor_config.json", "chat_template.jinja", "generation_config.json", "special_tokens_map.json", "merges.txt", "vocab.json", "added_tokens.json", "tokenizer.json", } SKIP_PARTS = ("assets/", "examples/", "neural_codec/") MAX_BYTES = 60_000_000 # keep the loading set light; skip multi-hundred-MB tarballs def want(path: str, size: int) -> bool: if any(path.startswith(p) or f"/{p}" in path for p in SKIP_PARTS): return False name = path.rsplit("/", 1)[-1] if path.endswith((".tar", ".png", ".jpg", ".jpeg", ".mp4", ".gif")): return False if size > MAX_BYTES: return False return name in KEEP_NAMES or path.endswith(KEEP_SUFFIXES) def sha256_of(p: Path) -> str: h = hashlib.sha256() with open(p, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def main() -> int: token = os.environ.get("HF_TOKEN") manifest = json.loads((ROOT / "weights" / "source_files.json").read_text()) results = {} for key, src in SOURCE_MODELS.items(): entry = manifest["sources"].get(key, {}) files = entry.get("files", []) got, skipped, verified = 0, 0, 0 for f in files: path, size = f["path"], f["size"] or 0 if f["is_weight"] or not want(path, size): skipped += 1 continue dest = OUT / key / path dest.parent.mkdir(parents=True, exist_ok=True) url = f"https://huggingface.co/{src.repo}/resolve/{src.revision}/{path}" req = urllib.request.Request(url, headers={"User-Agent": "noeon/0.1"}) if token: req.add_header("Authorization", f"Bearer {token}") with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as out: out.write(r.read()) # verify: blob sha1 for small git files is the git object id got += 1 results[key] = {"downloaded": got, "skipped": skipped} print(f"[{key}] downloaded {got} loading files, skipped {skipped} " f"(weights/assets/codec)") (OUT / "_fetch_summary.json").write_text(json.dumps(results, indent=2)) print(f"\n-> {OUT}") return 0 if __name__ == "__main__": raise SystemExit(main())