Spaces:
Configuration error
Configuration error
File size: 1,744 Bytes
151ec26 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | from pathlib import Path
from huggingface_hub import snapshot_download
BASE_DIR = Path(__file__).resolve().parents[1]
MODEL_DIR = BASE_DIR / "models" / "pretrained"
MODELS = {
"trocr": "microsoft/trocr-base-printed",
"biomedbert":
"microsoft/BiomedNLP-BiomedBERT-base-uncased-abstract-fulltext",
"sapbert":
"cambridgeltl/SapBERT-from-PubMedBERT-fulltext",
"intent":
"distilbert/distilbert-base-uncased",
"whisper":
"openai/whisper-base",
"indictrans_en_indic":
"ai4bharat/indictrans2-en-indic-dist-200M",
"indictrans_indic_indic":
"ai4bharat/indictrans2-indic-indic-dist-320M",
}
# Only download files required for PyTorch/Transformers.
# This avoids downloading TensorFlow, Flax and duplicate PyTorch weights.
ALLOW_PATTERNS = [
"*.json",
"*.safetensors",
"*.txt",
"*.model",
"*.vocab",
"*.merges",
"*.tiktoken",
"*.py",
]
for name, repo_id in MODELS.items():
destination = MODEL_DIR / name
print("\n" + "=" * 70)
print(f"Downloading: {name}")
print(f"Repository: {repo_id}")
print(f"Destination: {destination}")
print("=" * 70)
destination.mkdir(
parents=True,
exist_ok=True
)
try:
snapshot_download(
repo_id=repo_id,
local_dir=str(destination),
allow_patterns=ALLOW_PATTERNS,
max_workers=2,
)
print(f"\n✅ Finished: {name}")
except Exception as e:
print(f"\n❌ FAILED: {name}")
print(f"Error: {e}")
print(
"\nThe script will continue with the next model."
)
print("\n" + "=" * 70)
print("MODEL DOWNLOAD PROCESS FINISHED")
print("=" * 70)
|