Karl Qiu commited on
Commit
a8d5f96
Β·
1 Parent(s): 7912365

Pre-download model weights at startup and increase GPU duration

Browse files
Files changed (2) hide show
  1. app.py +20 -9
  2. llm_reviewer.py +49 -26
app.py CHANGED
@@ -105,26 +105,37 @@ def classify_comment(comment: str, strategy: str):
105
 
106
  except Exception as e:
107
  tb = traceback.format_exc()
108
- err = str(e).lower()
 
 
 
 
 
 
 
 
 
 
 
109
 
110
- # ── GPU / quota / auth errors β†’ helpful message ───────
111
- if any(k in err for k in [
112
- "gpu", "quota", "zero", "login", "sign", "unauthorized",
113
- ]) or (isinstance(e, RuntimeError) and "gpu" not in err):
114
  return (
115
  "## ⚠️ GPU Unavailable",
116
  "This can happen if:\n"
117
  "- You are not signed in to HuggingFace "
118
  "([sign in here](https://huggingface.co/login) β€” free)\n"
119
  "- Your daily ZeroGPU quota is exhausted\n"
120
- "- The GPU is temporarily busy\n\n"
 
 
121
  "**ML-only works without GPU** β€” select "
122
  "**ML-only (CPU, no login needed)** above.\n\n"
123
- f"<details><summary>Technical details</summary>\n\n"
124
- f"```\n{tb}\n```\n</details>",
 
 
125
  )
126
 
127
- # ── All other errors β†’ show full traceback ────────────
128
  return "## ❌ Error", f"```\n{tb}\n```"
129
 
130
 
 
105
 
106
  except Exception as e:
107
  tb = traceback.format_exc()
108
+ err_type = type(e).__name__
109
+ err_msg = str(e)
110
+
111
+ # ── ZeroGPU wraps the real error β†’ guide user to logs ─
112
+ gpu_hints = [
113
+ "gpu", "quota", "zero", "login", "sign",
114
+ "unauthorized", "ZeroGPUTask",
115
+ ]
116
+ is_gpu_error = (
117
+ any(k.lower() in (err_type + err_msg).lower() for k in gpu_hints)
118
+ or isinstance(e, RuntimeError)
119
+ )
120
 
121
+ if is_gpu_error:
 
 
 
122
  return (
123
  "## ⚠️ GPU Unavailable",
124
  "This can happen if:\n"
125
  "- You are not signed in to HuggingFace "
126
  "([sign in here](https://huggingface.co/login) β€” free)\n"
127
  "- Your daily ZeroGPU quota is exhausted\n"
128
+ "- The model is still loading on first call "
129
+ "(try again in ~30 s)\n"
130
+ "- The GPU timed out during model loading\n\n"
131
  "**ML-only works without GPU** β€” select "
132
  "**ML-only (CPU, no login needed)** above.\n\n"
133
+ f"<details><summary>Error: {err_type}: {err_msg}"
134
+ f"</summary>\n\n```\n{tb}\n```\n\n"
135
+ "Check full logs: Space Settings β†’ Logs"
136
+ "\n</details>",
137
  )
138
 
 
139
  return "## ❌ Error", f"```\n{tb}\n```"
140
 
141
 
llm_reviewer.py CHANGED
@@ -1,12 +1,18 @@
1
  """
2
  llm_reviewer.py β€” LLM reviewer for ambiguous comments (Stage 2).
3
 
4
- Qwen2.5-14B-Instruct is loaded lazily in NF4 quantization via
5
- bitsandbytes (~8 GB VRAM) on the first ZeroGPU call. The A100
6
- handles this comfortably alongside DistilBERT on CPU.
7
 
8
- The model stays in memory after first load, so subsequent calls
9
- skip the loading step and run inference directly (~1–2 s).
 
 
 
 
 
 
 
 
10
 
11
  If the `spaces` package is unavailable (local testing), the GPU
12
  decorator becomes a no-op so the code still works on any machine.
@@ -29,6 +35,7 @@ except ImportError:
29
 
30
  spaces = _SpacesFallback()
31
 
 
32
  import torch
33
  from typing import Tuple
34
  from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
@@ -50,18 +57,34 @@ _SYSTEM_PROMPT = (
50
  "Do not explain."
51
  )
52
 
53
- # ── Lazy-loaded model state ───────────────────────────────────
54
- # The model is loaded on the first @spaces.GPU call, not at
55
- # import time. This avoids startup failures when GPU is not
56
- # yet allocated and keeps Space boot time fast.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  _model = None
58
- _tokenizer = None
59
  _load_error = None
60
 
61
 
62
  def _ensure_model_loaded():
63
- """Load model and tokenizer once, on first use inside a GPU context."""
64
- global _model, _tokenizer, _load_error
65
 
66
  if _model is not None:
67
  return True
@@ -69,15 +92,8 @@ def _ensure_model_loaded():
69
  return False
70
 
71
  try:
72
- print(f"⏳ Loading {LLM_MODEL_NAME} ({LLM_QUANTIZATION}) …")
73
 
74
- _tokenizer = AutoTokenizer.from_pretrained(
75
- LLM_MODEL_NAME, trust_remote_code=True
76
- )
77
- if _tokenizer.pad_token is None:
78
- _tokenizer.pad_token = _tokenizer.eos_token
79
-
80
- # ── Build quantization config ─────────────────────────
81
  load_kwargs = {
82
  "trust_remote_code": True,
83
  "device_map": "auto",
@@ -90,10 +106,10 @@ def _ensure_model_loaded():
90
  bnb_4bit_compute_dtype=torch.float16,
91
  bnb_4bit_use_double_quant=True,
92
  )
93
- print(" NF4 quantization via bitsandbytes")
94
  else:
95
  load_kwargs["torch_dtype"] = torch.float16
96
- print(" float16 (no quantization)")
97
 
98
  _model = AutoModelForCausalLM.from_pretrained(
99
  LLM_MODEL_NAME, **load_kwargs
@@ -108,7 +124,8 @@ def _ensure_model_loaded():
108
 
109
  except Exception as e:
110
  _load_error = str(e)
111
- print(f"❌ Failed to load {LLM_MODEL_NAME}: {_load_error}")
 
112
  return False
113
 
114
 
@@ -124,14 +141,20 @@ def _parse_answer(answer: str) -> Tuple[str, float]:
124
  return (DECISION_BLOCK, 0.50)
125
 
126
 
127
- # ── GPU-decorated classification function ─────────────────────
128
- @spaces.GPU(duration=120)
 
 
 
 
 
 
129
  def gpu_classify(text: str) -> Tuple[str, float, str]:
130
  """
131
  Classify one comment with the LLM.
132
 
133
  On HuggingFace Spaces this runs on a ZeroGPU A100 slice.
134
- The first call includes model loading (~30–40 s for 14B NF4);
135
  subsequent calls run inference only (~1–2 s).
136
  """
137
  if not _ensure_model_loaded():
 
1
  """
2
  llm_reviewer.py β€” LLM reviewer for ambiguous comments (Stage 2).
3
 
4
+ Qwen2.5-14B-Instruct is served via ZeroGPU in two phases:
 
 
5
 
6
+ Startup (CPU, no GPU needed):
7
+ β€’ Download model weights to local cache (~8 GB, once)
8
+ β€’ Load tokenizer
9
+
10
+ First @spaces.GPU call:
11
+ β€’ Load cached weights into GPU with NF4 quantization (~30–40 s)
12
+ β€’ Run inference (~1–2 s)
13
+
14
+ Subsequent @spaces.GPU calls:
15
+ β€’ Model already in memory β†’ inference only (~1–2 s)
16
 
17
  If the `spaces` package is unavailable (local testing), the GPU
18
  decorator becomes a no-op so the code still works on any machine.
 
35
 
36
  spaces = _SpacesFallback()
37
 
38
+ import traceback
39
  import torch
40
  from typing import Tuple
41
  from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 
57
  "Do not explain."
58
  )
59
 
60
+ # ══════════════════════════════════════════════════════════════
61
+ # STARTUP PHASE (CPU only β€” no GPU, no quota consumed)
62
+ # ══════════════════════════════════════════════════════════════
63
+ # Pre-download model weights into the HuggingFace cache so that
64
+ # the @spaces.GPU function only needs to load from local disk
65
+ # (seconds) instead of downloading from the Hub (minutes).
66
+
67
+ print(f"⏳ Pre-downloading {LLM_MODEL_NAME} weights …")
68
+ from huggingface_hub import snapshot_download
69
+ snapshot_download(LLM_MODEL_NAME)
70
+ print(f"βœ… Model weights cached on disk")
71
+
72
+ print(f"⏳ Loading tokenizer …")
73
+ _tokenizer = AutoTokenizer.from_pretrained(
74
+ LLM_MODEL_NAME, trust_remote_code=True
75
+ )
76
+ if _tokenizer.pad_token is None:
77
+ _tokenizer.pad_token = _tokenizer.eos_token
78
+ print(f"βœ… Tokenizer ready")
79
+
80
+ # ── Lazy-loaded model (populated on first GPU call) ───────────
81
  _model = None
 
82
  _load_error = None
83
 
84
 
85
  def _ensure_model_loaded():
86
+ """Load model from local cache into GPU memory (first call only)."""
87
+ global _model, _load_error
88
 
89
  if _model is not None:
90
  return True
 
92
  return False
93
 
94
  try:
95
+ print(f"⏳ Loading {LLM_MODEL_NAME} into GPU with NF4 …")
96
 
 
 
 
 
 
 
 
97
  load_kwargs = {
98
  "trust_remote_code": True,
99
  "device_map": "auto",
 
106
  bnb_4bit_compute_dtype=torch.float16,
107
  bnb_4bit_use_double_quant=True,
108
  )
109
+ print(" Using NF4 quantization via bitsandbytes")
110
  else:
111
  load_kwargs["torch_dtype"] = torch.float16
112
+ print(" Using float16 (no quantization)")
113
 
114
  _model = AutoModelForCausalLM.from_pretrained(
115
  LLM_MODEL_NAME, **load_kwargs
 
124
 
125
  except Exception as e:
126
  _load_error = str(e)
127
+ print(f"❌ Failed to load model: {_load_error}")
128
+ traceback.print_exc()
129
  return False
130
 
131
 
 
141
  return (DECISION_BLOCK, 0.50)
142
 
143
 
144
+ # ══════════════════════════════════════════════════════════════
145
+ # GPU INFERENCE (ZeroGPU A100, quota consumed per call)
146
+ # ═══════���══════════════════════════════════════════════════════
147
+ # duration=180 gives enough headroom for first-call model loading
148
+ # (~30–40 s) plus inference (~1–2 s). Subsequent calls finish in
149
+ # seconds but the unused budget is returned immediately.
150
+
151
+ @spaces.GPU(duration=180)
152
  def gpu_classify(text: str) -> Tuple[str, float, str]:
153
  """
154
  Classify one comment with the LLM.
155
 
156
  On HuggingFace Spaces this runs on a ZeroGPU A100 slice.
157
+ First call loads the model from local cache (~30–40 s);
158
  subsequent calls run inference only (~1–2 s).
159
  """
160
  if not _ensure_model_loaded():