Anmol2507 commited on
Commit
b9f0050
·
verified ·
1 Parent(s): 72ab9a4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +46 -43
app.py CHANGED
@@ -7,7 +7,7 @@ from peft import PeftModel
7
 
8
 
9
  # ============================================================
10
- # CONFIG
11
  # ============================================================
12
 
13
  BASE_MODEL = "Qwen/Qwen2.5-7B-Instruct"
@@ -18,7 +18,7 @@ MAX_NEW_TOKENS = 32
18
 
19
 
20
  # ============================================================
21
- # TOKENIZER
22
  # ============================================================
23
 
24
  print("Loading tokenizer...")
@@ -34,10 +34,8 @@ print("Tokenizer loaded.")
34
  # ============================================================
35
  # LOAD BASE MODEL
36
  # ============================================================
37
- # Important for ZeroGPU:
38
- # The real GPU is not attached during application startup.
39
- # Therefore the base model is initially loaded on CPU.
40
- # Later, model.to("cuda") registers it with ZeroGPU.
41
  # ============================================================
42
 
43
  print("Loading base model...")
@@ -56,9 +54,9 @@ print("Base model loaded.")
56
  # ============================================================
57
  # LOAD LoRA ADAPTER
58
  # ============================================================
59
- # IMPORTANT:
60
- # torch_device="cpu" prevents PEFT from trying to load the
61
- # adapter weights directly onto CUDA during startup.
62
  # ============================================================
63
 
64
  print("Loading LoRA adapter...")
@@ -77,11 +75,12 @@ print("LoRA adapter loaded.")
77
  # ============================================================
78
  # REGISTER MODEL WITH ZEROGPU
79
  # ============================================================
80
- # This MUST remain at module scope.
81
- # ZeroGPU intercepts this CUDA placement and manages the
82
- # model weights for GPU workers.
83
  # ============================================================
84
 
 
 
85
  model = model.to("cuda")
86
 
87
  model.eval()
@@ -120,15 +119,9 @@ def build_prompt(problem, answer_a, answer_b):
120
  {
121
  "role": "user",
122
  "content": (
123
- f"Problem:\n"
124
- f"{problem}\n\n"
125
-
126
- f"Answer A:\n"
127
- f"{answer_a}\n\n"
128
-
129
- f"Answer B:\n"
130
- f"{answer_b}\n\n"
131
-
132
  "Evaluate both answers."
133
  )
134
  }
@@ -142,14 +135,14 @@ def build_prompt(problem, answer_a, answer_b):
142
 
143
 
144
  # ============================================================
145
- # JUDGE FUNCTION
146
  # ============================================================
147
 
148
  @spaces.GPU(duration=60)
149
  def judge(problem, answer_a, answer_b):
150
 
151
  # --------------------------------------------------------
152
- # Validate input
153
  # --------------------------------------------------------
154
 
155
  if not problem or not problem.strip():
@@ -163,7 +156,7 @@ def judge(problem, answer_a, answer_b):
163
 
164
 
165
  # --------------------------------------------------------
166
- # Build prompt
167
  # --------------------------------------------------------
168
 
169
  prompt = build_prompt(
@@ -186,7 +179,7 @@ def judge(problem, answer_a, answer_b):
186
 
187
 
188
  # --------------------------------------------------------
189
- # Move inputs to CUDA
190
  # --------------------------------------------------------
191
 
192
  inputs = {
@@ -196,7 +189,7 @@ def judge(problem, answer_a, answer_b):
196
 
197
 
198
  # --------------------------------------------------------
199
- # Generate
200
  # --------------------------------------------------------
201
 
202
  with torch.inference_mode():
@@ -210,7 +203,7 @@ def judge(problem, answer_a, answer_b):
210
 
211
 
212
  # --------------------------------------------------------
213
- # Extract only generated tokens
214
  # --------------------------------------------------------
215
 
216
  input_length = inputs["input_ids"].shape[1]
@@ -219,7 +212,7 @@ def judge(problem, answer_a, answer_b):
219
 
220
 
221
  # --------------------------------------------------------
222
- # Decode
223
  # --------------------------------------------------------
224
 
225
  response = tokenizer.decode(
@@ -232,13 +225,17 @@ def judge(problem, answer_a, answer_b):
232
 
233
 
234
  # ============================================================
235
- # GRADIO UI
236
  # ============================================================
237
 
238
  with gr.Blocks(
239
  title="LLM as Judge"
240
  ) as demo:
241
 
 
 
 
 
242
  gr.Markdown(
243
  """
244
  # 🧑‍⚖️ LLM as Judge
@@ -284,7 +281,7 @@ with gr.Blocks(
284
 
285
 
286
  # --------------------------------------------------------
287
- # Button
288
  # --------------------------------------------------------
289
 
290
  judge_button = gr.Button(
@@ -304,22 +301,28 @@ with gr.Blocks(
304
 
305
 
306
  # --------------------------------------------------------
307
- # Connect button
 
 
 
 
308
  # --------------------------------------------------------
309
 
310
- judge_button.click(
311
- fn=judge,
312
- inputs=[
313
- problem,
314
- answer_a,
315
- answer_b
316
- ],
317
- outputs=result,
318
- api_name="judge"
319
- )
 
320
 
321
  # ============================================================
322
- # LAUNCH
323
  # ============================================================
324
 
325
- demo.launch()
 
 
7
 
8
 
9
  # ============================================================
10
+ # CONFIGURATION
11
  # ============================================================
12
 
13
  BASE_MODEL = "Qwen/Qwen2.5-7B-Instruct"
 
18
 
19
 
20
  # ============================================================
21
+ # LOAD TOKENIZER
22
  # ============================================================
23
 
24
  print("Loading tokenizer...")
 
34
  # ============================================================
35
  # LOAD BASE MODEL
36
  # ============================================================
37
+ # Load on CPU first because ZeroGPU does not provide a real
38
+ # CUDA device during application startup.
 
 
39
  # ============================================================
40
 
41
  print("Loading base model...")
 
54
  # ============================================================
55
  # LOAD LoRA ADAPTER
56
  # ============================================================
57
+ # torch_device="cpu" is important.
58
+ # It prevents PEFT from trying to load adapter weights onto
59
+ # CUDA before ZeroGPU allocates a GPU.
60
  # ============================================================
61
 
62
  print("Loading LoRA adapter...")
 
75
  # ============================================================
76
  # REGISTER MODEL WITH ZEROGPU
77
  # ============================================================
78
+ # Keep this at module level.
79
+ # ZeroGPU intercepts this CUDA placement.
 
80
  # ============================================================
81
 
82
+ print("Registering model with ZeroGPU...")
83
+
84
  model = model.to("cuda")
85
 
86
  model.eval()
 
119
  {
120
  "role": "user",
121
  "content": (
122
+ f"Problem:\n{problem}\n\n"
123
+ f"Answer A:\n{answer_a}\n\n"
124
+ f"Answer B:\n{answer_b}\n\n"
 
 
 
 
 
 
125
  "Evaluate both answers."
126
  )
127
  }
 
135
 
136
 
137
  # ============================================================
138
+ # JUDGE / INFERENCE FUNCTION
139
  # ============================================================
140
 
141
  @spaces.GPU(duration=60)
142
  def judge(problem, answer_a, answer_b):
143
 
144
  # --------------------------------------------------------
145
+ # Validate inputs
146
  # --------------------------------------------------------
147
 
148
  if not problem or not problem.strip():
 
156
 
157
 
158
  # --------------------------------------------------------
159
+ # Create prompt
160
  # --------------------------------------------------------
161
 
162
  prompt = build_prompt(
 
179
 
180
 
181
  # --------------------------------------------------------
182
+ # Move input tensors to GPU
183
  # --------------------------------------------------------
184
 
185
  inputs = {
 
189
 
190
 
191
  # --------------------------------------------------------
192
+ # Generate response
193
  # --------------------------------------------------------
194
 
195
  with torch.inference_mode():
 
203
 
204
 
205
  # --------------------------------------------------------
206
+ # Remove prompt tokens
207
  # --------------------------------------------------------
208
 
209
  input_length = inputs["input_ids"].shape[1]
 
212
 
213
 
214
  # --------------------------------------------------------
215
+ # Decode model response
216
  # --------------------------------------------------------
217
 
218
  response = tokenizer.decode(
 
225
 
226
 
227
  # ============================================================
228
+ # GRADIO APPLICATION
229
  # ============================================================
230
 
231
  with gr.Blocks(
232
  title="LLM as Judge"
233
  ) as demo:
234
 
235
+ # --------------------------------------------------------
236
+ # Header
237
+ # --------------------------------------------------------
238
+
239
  gr.Markdown(
240
  """
241
  # 🧑‍⚖️ LLM as Judge
 
281
 
282
 
283
  # --------------------------------------------------------
284
+ # Judge button
285
  # --------------------------------------------------------
286
 
287
  judge_button = gr.Button(
 
301
 
302
 
303
  # --------------------------------------------------------
304
+ # API EVENT
305
+ # --------------------------------------------------------
306
+ # IMPORTANT:
307
+ # This must remain INSIDE the Blocks context.
308
+ # api_name="judge" exposes the endpoint as /judge.
309
  # --------------------------------------------------------
310
 
311
+ judge_button.click(
312
+ fn=judge,
313
+ inputs=[
314
+ problem,
315
+ answer_a,
316
+ answer_b
317
+ ],
318
+ outputs=result,
319
+ api_name="judge"
320
+ )
321
+
322
 
323
  # ============================================================
324
+ # START APPLICATION
325
  # ============================================================
326
 
327
+ if __name__ == "__main__":
328
+ demo.launch()