VerdaSense-Dressing / fetch_models.py
qijingtee1227's picture
VerdaSense research retrieval (frozen V6-RET-C1), code only; KB not included
0270590
Raw History Blame Contribute Delete
2.97 kB
"""Build step: download the two pinned public models (and the reviewed new-impl code) and verify every file.
BGE goes into the HF cache (the frozen Encoder finds it by revision, offline). GTE and its code go to fixed folders.
"""
import hashlib
import shutil
from pathlib import Path
from huggingface_hub import hf_hub_download, snapshot_download
BGE = ('BAAI/bge-large-en-v1.5', 'd4aa6901d3a41ba39fb536a557fa166f842b0e09', {
'model.safetensors': '45e1954914e29bd74080e6c1510165274ff5279421c89f76c418878732f64ae7',
'1_Pooling/config.json': 'e54c164a07274f2eb45bb724f54a79d1efcc90c41573887cd9a29aeee0597352',
'config.json': '446712fac367857b4b1302762fe1cd7bfa8b3c4b77b4dc5d77c4025407660896',
'config_sentence_transformers.json': '940d5f50db195fa6e5e6a4f122c095f77880de259d74b14a65779ed48bdd7c56',
'modules.json': '84e40c8e006c9b1d6c122e02cba9b02458120b5fb0c87b746c41e0207cf642cf',
'sentence_bert_config.json': '84e39fda68ccbff05bfa723ae9c0e70e23e2ec373b76e0f8c6e71af72a693cbf',
'special_tokens_map.json': 'b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3',
'tokenizer.json': 'd241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66',
'tokenizer_config.json': '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3'})
GTE = ('Alibaba-NLP/gte-multilingual-reranker-base', '8215cf04918ba6f7b6a62bb44238ce2953d8831c', {
'config.json': '995730781d157e147c13ccdfe0eb20a0875c486b6c4de8c97f0bbd845549dbc0',
'model.safetensors': '10ebaa49322dd7e01a13a91c49810939e3f91f231aceaa47fdf0cab3083954f6',
'special_tokens_map.json': '8c785abebea9ae3257b61681b4e6fd8365ceafde980c21970d001e834cf10835',
'tokenizer.json': 'd6f76fe13d42f80dcee0cb86a1aeb5f14f8909bb8a8782f7a4a4ad76697ef164',
'tokenizer_config.json': '6f00514620aff01ba8b7291b2394e98daca5be264cb743805232d9ae27494b2a'})
CODE = ('Alibaba-NLP/new-impl', '40ced75c3017eb27626c9d4ea981bde21a2662f4', {
'configuration.py': '3411088045ffb8a9a0aa9936eae275896b39983a2ee5b08f091b44e6289e4fe4',
'modeling.py': '374670b416fcc82f081c9cd28b5fd61c2bd91bbe18eb4798fcc48a81f9c250a0'})
def sha(p):
h = hashlib.sha256()
with open(p, 'rb') as f:
for b in iter(lambda: f.read(1 << 20), b''):
h.update(b)
return h.hexdigest()
def check(folder, files, what):
for name, digest in files.items():
actual = sha(Path(folder) / name)
if actual != digest:
raise SystemExit(f'{what}: {name} does not match the pinned SHA-256')
print(what, 'verified', len(files), 'files')
repo, rev, files = BGE
check(snapshot_download(repo, revision=rev, allow_patterns=list(files)), files, 'BGE ' + rev[:7])
for (repo, rev, files), dest in ((GTE, '/srv/gte/snapshot'), (CODE, '/srv/gte/code')):
Path(dest).mkdir(parents=True, exist_ok=True)
for name in files:
shutil.copyfile(hf_hub_download(repo, name, revision=rev), Path(dest) / name)
check(dest, files, f'{repo}@{rev[:7]}')