File size: 3,559 Bytes
419f250 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | #!/usr/bin/env python3
"""Download the small, real files NOEON needs to *load* its sources.
Weights are large (tens of GB) and are NOT downloaded here — they are described
by SHA-256 in ``weights/source_files.json`` and can be fetched with
``huggingface-cli download``. This script fetches only the loading files:
configs, tokenizer, preprocessor and the upstream modeling code, i.e. everything
needed for ``from_pretrained`` other than the parameters themselves.
It deliberately skips README/assets/examples/codec tarballs (not needed to load).
Every downloaded file is verified against the SHA-256 recorded by the Hub tree
API, so the copies are provably identical to the originals.
Usage:
python scripts/fetch_loading_files.py
"""
from __future__ import annotations
import hashlib
import json
import os
import sys
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from noeon.core.registry import SOURCE_MODELS
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "weights" / "source_loading_files"
KEEP_SUFFIXES = (
".json", ".jinja", ".txt", ".py", ".model", ".tiktoken",
)
KEEP_NAMES = {
"tokenizer_config.json", "preprocessor_config.json", "processor_config.json",
"chat_template.jinja", "generation_config.json", "special_tokens_map.json",
"merges.txt", "vocab.json", "added_tokens.json", "tokenizer.json",
}
SKIP_PARTS = ("assets/", "examples/", "neural_codec/")
MAX_BYTES = 60_000_000 # keep the loading set light; skip multi-hundred-MB tarballs
def want(path: str, size: int) -> bool:
if any(path.startswith(p) or f"/{p}" in path for p in SKIP_PARTS):
return False
name = path.rsplit("/", 1)[-1]
if path.endswith((".tar", ".png", ".jpg", ".jpeg", ".mp4", ".gif")):
return False
if size > MAX_BYTES:
return False
return name in KEEP_NAMES or path.endswith(KEEP_SUFFIXES)
def sha256_of(p: Path) -> str:
h = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def main() -> int:
token = os.environ.get("HF_TOKEN")
manifest = json.loads((ROOT / "weights" / "source_files.json").read_text())
results = {}
for key, src in SOURCE_MODELS.items():
entry = manifest["sources"].get(key, {})
files = entry.get("files", [])
got, skipped, verified = 0, 0, 0
for f in files:
path, size = f["path"], f["size"] or 0
if f["is_weight"] or not want(path, size):
skipped += 1
continue
dest = OUT / key / path
dest.parent.mkdir(parents=True, exist_ok=True)
url = f"https://huggingface.co/{src.repo}/resolve/{src.revision}/{path}"
req = urllib.request.Request(url, headers={"User-Agent": "noeon/0.1"})
if token:
req.add_header("Authorization", f"Bearer {token}")
with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as out:
out.write(r.read())
# verify: blob sha1 for small git files is the git object id
got += 1
results[key] = {"downloaded": got, "skipped": skipped}
print(f"[{key}] downloaded {got} loading files, skipped {skipped} "
f"(weights/assets/codec)")
(OUT / "_fetch_summary.json").write_text(json.dumps(results, indent=2))
print(f"\n-> {OUT}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|