scripts/download_source.py

"""Download the pinned official checkpoint and verify its published weight hash."""
import hashlib
import json
from pathlib import Path
import urllib.request

REPO = 'openbmb/MiniCPM5-2B'
REVISION = 'abe115e887989b14f05e64a3b260648329324c3f'
WEIGHT = 'model-00000-of-00001.safetensors'
EXPECTED = '14fb8e7f0a18d53d1f239773758bf581cee7e456a4523a54622c3a245b64402c'
out = Path('models/source'); out.mkdir(parents=True, exist_ok=True)
files = ['config.json','generation_config.json','tokenizer.json','tokenizer_config.json',
         'special_tokens_map.json','chat_template.jinja','model.safetensors.index.json',WEIGHT]
manifest = {'repo':REPO,'revision':REVISION,'files':{}}
for name in files:
    path = out / name
    if not path.exists():
        temporary = out / (name + '.partial')
        with urllib.request.urlopen(f'https://huggingface.co/{REPO}/resolve/{REVISION}/{name}', timeout=120) as response, temporary.open('wb') as f:
            while data := response.read(8*1024**2): f.write(data)
        temporary.rename(path)
    h = hashlib.sha256()
    with path.open('rb') as f:
        while data := f.read(8*1024**2): h.update(data)
    if name == WEIGHT: assert h.hexdigest() == EXPECTED, 'Official weight checksum mismatch'
    manifest['files'][name] = {'bytes':path.stat().st_size,'sha256':h.hexdigest()}
    print(name, h.hexdigest(), flush=True)
(out/'provenance.json').write_text(json.dumps(manifest,indent=2))