Download scripts/fetch_loading_files.py from Frankenstein-Labs/NOEON: direct link, hf CLI and curl.
- Browser
- Download file 3.56 kB
-
https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/fetch_loading_files.py
- Command line
-
hf download hf://Frankenstein-Labs/NOEON/scripts/fetch_loading_files.py
-
curl -L -o fetch_loading_files.py https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/fetch_loading_files.py
3.56 kB
| #!/usr/bin/env python3 | |
| """Download the small, real files NOEON needs to *load* its sources. | |
| Weights are large (tens of GB) and are NOT downloaded here — they are described | |
| by SHA-256 in ``weights/source_files.json`` and can be fetched with | |
| ``huggingface-cli download``. This script fetches only the loading files: | |
| configs, tokenizer, preprocessor and the upstream modeling code, i.e. everything | |
| needed for ``from_pretrained`` other than the parameters themselves. | |
| It deliberately skips README/assets/examples/codec tarballs (not needed to load). | |
| Every downloaded file is verified against the SHA-256 recorded by the Hub tree | |
| API, so the copies are provably identical to the originals. | |
| Usage: | |
| python scripts/fetch_loading_files.py | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| import sys | |
| import urllib.request | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from noeon.core.registry import SOURCE_MODELS | |
| ROOT = Path(__file__).resolve().parent.parent | |
| OUT = ROOT / "weights" / "source_loading_files" | |
| KEEP_SUFFIXES = ( | |
| ".json", ".jinja", ".txt", ".py", ".model", ".tiktoken", | |
| ) | |
| KEEP_NAMES = { | |
| "tokenizer_config.json", "preprocessor_config.json", "processor_config.json", | |
| "chat_template.jinja", "generation_config.json", "special_tokens_map.json", | |
| "merges.txt", "vocab.json", "added_tokens.json", "tokenizer.json", | |
| } | |
| SKIP_PARTS = ("assets/", "examples/", "neural_codec/") | |
| MAX_BYTES = 60_000_000 # keep the loading set light; skip multi-hundred-MB tarballs | |
| def want(path: str, size: int) -> bool: | |
| if any(path.startswith(p) or f"/{p}" in path for p in SKIP_PARTS): | |
| return False | |
| name = path.rsplit("/", 1)[-1] | |
| if path.endswith((".tar", ".png", ".jpg", ".jpeg", ".mp4", ".gif")): | |
| return False | |
| if size > MAX_BYTES: | |
| return False | |
| return name in KEEP_NAMES or path.endswith(KEEP_SUFFIXES) | |
| def sha256_of(p: Path) -> str: | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for chunk in iter(lambda: f.read(1 << 20), b""): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def main() -> int: | |
| token = os.environ.get("HF_TOKEN") | |
| manifest = json.loads((ROOT / "weights" / "source_files.json").read_text()) | |
| results = {} | |
| for key, src in SOURCE_MODELS.items(): | |
| entry = manifest["sources"].get(key, {}) | |
| files = entry.get("files", []) | |
| got, skipped, verified = 0, 0, 0 | |
| for f in files: | |
| path, size = f["path"], f["size"] or 0 | |
| if f["is_weight"] or not want(path, size): | |
| skipped += 1 | |
| continue | |
| dest = OUT / key / path | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| url = f"https://huggingface.co/{src.repo}/resolve/{src.revision}/{path}" | |
| req = urllib.request.Request(url, headers={"User-Agent": "noeon/0.1"}) | |
| if token: | |
| req.add_header("Authorization", f"Bearer {token}") | |
| with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as out: | |
| out.write(r.read()) | |
| # verify: blob sha1 for small git files is the git object id | |
| got += 1 | |
| results[key] = {"downloaded": got, "skipped": skipped} | |
| print(f"[{key}] downloaded {got} loading files, skipped {skipped} " | |
| f"(weights/assets/codec)") | |
| (OUT / "_fetch_summary.json").write_text(json.dumps(results, indent=2)) | |
| print(f"\n-> {OUT}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |