"""Build step: download the two pinned public models (and the reviewed new-impl code) and verify every file. BGE goes into the HF cache (the frozen Encoder finds it by revision, offline). GTE and its code go to fixed folders. """ import hashlib import shutil from pathlib import Path from huggingface_hub import hf_hub_download, snapshot_download BGE = ('BAAI/bge-large-en-v1.5', 'd4aa6901d3a41ba39fb536a557fa166f842b0e09', { 'model.safetensors': '45e1954914e29bd74080e6c1510165274ff5279421c89f76c418878732f64ae7', '1_Pooling/config.json': 'e54c164a07274f2eb45bb724f54a79d1efcc90c41573887cd9a29aeee0597352', 'config.json': '446712fac367857b4b1302762fe1cd7bfa8b3c4b77b4dc5d77c4025407660896', 'config_sentence_transformers.json': '940d5f50db195fa6e5e6a4f122c095f77880de259d74b14a65779ed48bdd7c56', 'modules.json': '84e40c8e006c9b1d6c122e02cba9b02458120b5fb0c87b746c41e0207cf642cf', 'sentence_bert_config.json': '84e39fda68ccbff05bfa723ae9c0e70e23e2ec373b76e0f8c6e71af72a693cbf', 'special_tokens_map.json': 'b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3', 'tokenizer.json': 'd241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66', 'tokenizer_config.json': '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3'}) GTE = ('Alibaba-NLP/gte-multilingual-reranker-base', '8215cf04918ba6f7b6a62bb44238ce2953d8831c', { 'config.json': '995730781d157e147c13ccdfe0eb20a0875c486b6c4de8c97f0bbd845549dbc0', 'model.safetensors': '10ebaa49322dd7e01a13a91c49810939e3f91f231aceaa47fdf0cab3083954f6', 'special_tokens_map.json': '8c785abebea9ae3257b61681b4e6fd8365ceafde980c21970d001e834cf10835', 'tokenizer.json': 'd6f76fe13d42f80dcee0cb86a1aeb5f14f8909bb8a8782f7a4a4ad76697ef164', 'tokenizer_config.json': '6f00514620aff01ba8b7291b2394e98daca5be264cb743805232d9ae27494b2a'}) CODE = ('Alibaba-NLP/new-impl', '40ced75c3017eb27626c9d4ea981bde21a2662f4', { 'configuration.py': '3411088045ffb8a9a0aa9936eae275896b39983a2ee5b08f091b44e6289e4fe4', 'modeling.py': '374670b416fcc82f081c9cd28b5fd61c2bd91bbe18eb4798fcc48a81f9c250a0'}) def sha(p): h = hashlib.sha256() with open(p, 'rb') as f: for b in iter(lambda: f.read(1 << 20), b''): h.update(b) return h.hexdigest() def check(folder, files, what): for name, digest in files.items(): actual = sha(Path(folder) / name) if actual != digest: raise SystemExit(f'{what}: {name} does not match the pinned SHA-256') print(what, 'verified', len(files), 'files') repo, rev, files = BGE check(snapshot_download(repo, revision=rev, allow_patterns=list(files)), files, 'BGE ' + rev[:7]) for (repo, rev, files), dest in ((GTE, '/srv/gte/snapshot'), (CODE, '/srv/gte/code')): Path(dest).mkdir(parents=True, exist_ok=True) for name in files: shutil.copyfile(hf_hub_download(repo, name, revision=rev), Path(dest) / name) check(dest, files, f'{repo}@{rev[:7]}')