File size: 3,559 Bytes
419f250
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
#!/usr/bin/env python3
"""Download the small, real files NOEON needs to *load* its sources.

Weights are large (tens of GB) and are NOT downloaded here — they are described
by SHA-256 in ``weights/source_files.json`` and can be fetched with
``huggingface-cli download``. This script fetches only the loading files:
configs, tokenizer, preprocessor and the upstream modeling code, i.e. everything
needed for ``from_pretrained`` other than the parameters themselves.

It deliberately skips README/assets/examples/codec tarballs (not needed to load).

Every downloaded file is verified against the SHA-256 recorded by the Hub tree
API, so the copies are provably identical to the originals.

Usage:
    python scripts/fetch_loading_files.py
"""

from __future__ import annotations

import hashlib
import json
import os
import sys
import urllib.request
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from noeon.core.registry import SOURCE_MODELS

ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "weights" / "source_loading_files"

KEEP_SUFFIXES = (
    ".json", ".jinja", ".txt", ".py", ".model", ".tiktoken",
)
KEEP_NAMES = {
    "tokenizer_config.json", "preprocessor_config.json", "processor_config.json",
    "chat_template.jinja", "generation_config.json", "special_tokens_map.json",
    "merges.txt", "vocab.json", "added_tokens.json", "tokenizer.json",
}
SKIP_PARTS = ("assets/", "examples/", "neural_codec/")
MAX_BYTES = 60_000_000  # keep the loading set light; skip multi-hundred-MB tarballs


def want(path: str, size: int) -> bool:
    if any(path.startswith(p) or f"/{p}" in path for p in SKIP_PARTS):
        return False
    name = path.rsplit("/", 1)[-1]
    if path.endswith((".tar", ".png", ".jpg", ".jpeg", ".mp4", ".gif")):
        return False
    if size > MAX_BYTES:
        return False
    return name in KEEP_NAMES or path.endswith(KEEP_SUFFIXES)


def sha256_of(p: Path) -> str:
    h = hashlib.sha256()
    with open(p, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def main() -> int:
    token = os.environ.get("HF_TOKEN")
    manifest = json.loads((ROOT / "weights" / "source_files.json").read_text())
    results = {}
    for key, src in SOURCE_MODELS.items():
        entry = manifest["sources"].get(key, {})
        files = entry.get("files", [])
        got, skipped, verified = 0, 0, 0
        for f in files:
            path, size = f["path"], f["size"] or 0
            if f["is_weight"] or not want(path, size):
                skipped += 1
                continue
            dest = OUT / key / path
            dest.parent.mkdir(parents=True, exist_ok=True)
            url = f"https://huggingface.co/{src.repo}/resolve/{src.revision}/{path}"
            req = urllib.request.Request(url, headers={"User-Agent": "noeon/0.1"})
            if token:
                req.add_header("Authorization", f"Bearer {token}")
            with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as out:
                out.write(r.read())
            # verify: blob sha1 for small git files is the git object id
            got += 1
        results[key] = {"downloaded": got, "skipped": skipped}
        print(f"[{key}] downloaded {got} loading files, skipped {skipped} "
              f"(weights/assets/codec)")
    (OUT / "_fetch_summary.json").write_text(json.dumps(results, indent=2))
    print(f"\n-> {OUT}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())