imbecome commited on
Commit
7bde357
·
verified ·
1 Parent(s): 58e6525

Update download_models.py

Browse files
Files changed (1) hide show
  1. download_models.py +45 -18
download_models.py CHANGED
@@ -1,34 +1,61 @@
1
  import os
2
  import sys
3
 
4
- # Path harus sinkron dengan Dockerfile
5
- HF_CACHE = os.getenv("HF_HOME", "/home/user/app/models_cache/transformers")
6
- WHISPER_CACHE = "/home/user/app/models_cache/whisper"
7
 
8
- os.makedirs(HF_CACHE, exist_ok=True)
9
- os.makedirs(WHISPER_CACHE, exist_ok=True)
 
10
 
11
- print("=== STARTING MODEL DOWNLOAD ===")
 
 
12
 
13
  try:
14
  from faster_whisper import WhisperModel
15
- print(f"Downloading Whisper tiny to {WHISPER_CACHE}...")
16
- WhisperModel("tiny", device="cpu", compute_type="int8", download_root=WHISPER_CACHE)
17
- print("Whisper model OK.")
 
 
 
 
 
 
 
 
 
18
  except Exception as e:
19
- print(f"ERROR Whisper: {e}")
20
  sys.exit(1)
21
 
 
 
 
 
22
  try:
23
- from transformers import AutoTokenizer, AutoModelForSequenceClassification
 
 
24
  model_name = "distilbert-base-uncased-finetuned-sst-2-english"
25
-
26
- print(f"Downloading {model_name} to {HF_CACHE}...")
27
- AutoTokenizer.from_pretrained(model_name, cache_dir=HF_CACHE)
28
- AutoModelForSequenceClassification.from_pretrained(model_name, cache_dir=HF_CACHE)
29
- print("Transformers models OK.")
 
 
 
 
 
 
 
 
30
  except Exception as e:
31
- print(f"ERROR Transformers: {e}")
32
  sys.exit(1)
33
 
34
- print("=== ALL MODELS READY ===")
 
 
 
1
  import os
2
  import sys
3
 
4
+ CACHE_DIR = "/home/user/app/models_cache"
5
+ os.makedirs(CACHE_DIR, exist_ok=True)
 
6
 
7
+ print("================================")
8
+ print("DOWNLOADING AI MODELS")
9
+ print("================================")
10
 
11
+ # ==========================
12
+ # WHISPER MODEL
13
+ # ==========================
14
 
15
  try:
16
  from faster_whisper import WhisperModel
17
+
18
+ print("Downloading Whisper tiny model...")
19
+
20
+ WhisperModel(
21
+ "tiny",
22
+ device="cpu",
23
+ compute_type="int8",
24
+ download_root=CACHE_DIR
25
+ )
26
+
27
+ print("Whisper download complete")
28
+
29
  except Exception as e:
30
+ print("WHISPER ERROR:", e)
31
  sys.exit(1)
32
 
33
+ # ==========================
34
+ # DISTILBERT SENTIMENT
35
+ # ==========================
36
+
37
  try:
38
+ from transformers import AutoTokenizer
39
+ from transformers import AutoModelForSequenceClassification
40
+
41
  model_name = "distilbert-base-uncased-finetuned-sst-2-english"
42
+
43
+ save_path = os.path.join(CACHE_DIR, "distilbert")
44
+
45
+ print("Downloading DistilBERT...")
46
+
47
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
48
+ model = AutoModelForSequenceClassification.from_pretrained(model_name)
49
+
50
+ tokenizer.save_pretrained(save_path)
51
+ model.save_pretrained(save_path)
52
+
53
+ print("DistilBERT saved")
54
+
55
  except Exception as e:
56
+ print("TRANSFORMER ERROR:", e)
57
  sys.exit(1)
58
 
59
+ print("================================")
60
+ print("ALL MODELS READY")
61
+ print("================================")