File size: 1,852 Bytes
4fcc185 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | """Acquire pinned official assets; existing files are verified, never silently replaced."""
import json,hashlib,urllib.request
from pathlib import Path
def acquire():
for name in ['reve-base','reve-positions']:
with urllib.request.urlopen('https://huggingface.co/api/models/brain-bzh/'+name,timeout=30) as response:access=json.load(response)
if access.get('gated') or access.get('private'):
Path('reports/model_access_blocker.json').write_text(json.dumps({'model':name,'gated':access.get('gated'),'private':access.get('private'),'action':'Stop; user must handle official license/access requirements. No alternate host, token or mirror will be used.'},indent=2))
raise PermissionError('Model access requires official authorization/license acceptance; see reports/model_access_blocker.json')
j=json.load(open(f'reports/{name}_files.json'));root=Path('models')/name;root.mkdir(exist_ok=True)
for e in j['siblings']:
n=e['rfilename']
if n=='.gitattributes':continue
p=root/n;p.parent.mkdir(parents=True,exist_ok=True)
if not p.exists():
with urllib.request.urlopen(f"https://huggingface.co/brain-bzh/{name}/resolve/{j['sha']}/{n}",timeout=120) as r,open(str(p)+'.part','wb') as out:
while b:=r.read(1024*1024):out.write(b)
Path(str(p)+'.part').replace(p)
digest=hashlib.sha256(p.read_bytes()).hexdigest()
if e.get('lfs'):assert digest==e['lfs']['sha256'],str(p)
print('Verified',p,p.stat().st_size,digest,flush=True)
p=Path('models/csbrain_pretrained.pth')
if not p.exists():
import gdown
assert gdown.download(id='1Z67je1HQhClyG9XER_zJzFMAKSVNmnF0',output=str(p),quiet=False,use_cookies=False)
sources=Path('reports/gpu_sources.json')
if sources.exists():assert hashlib.sha256(p.read_bytes()).hexdigest()==json.load(open(sources))['csbrain']['checkpoint_sha256']
if __name__=='__main__':acquire()
|