File size: 840 Bytes
6058321 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | from pathlib import Path
import json,hashlib,shutil
from huggingface_hub import hf_hub_download
root=Path(__file__).resolve().parent
for x in json.loads((root/'SHARED_ASSETS.json').read_text())['files']:
dest=root/x['destination']
def digest(p):
h=hashlib.sha256()
with p.open('rb') as f:
for b in iter(lambda:f.read(8*2**20),b''):h.update(b)
return h.hexdigest()
if dest.exists():
if dest.stat().st_size!=x['bytes'] or digest(dest)!=x['sha256']:raise RuntimeError('Existing file differs: '+str(dest))
continue
p=Path(hf_hub_download(repo_id=x['repo_id'],filename=x['path'],revision=x['revision']))
assert p.stat().st_size==x['bytes'] and digest(p)==x['sha256'],x['path']
dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest)
assert digest(dest)==x['sha256']
print('Verified restored',x['destination'])
|