rtferraz commited on
Commit
3fc0eb8
Β·
verified Β·
1 Parent(s): f46c90b

fix: eval crash (ZeroDivisionError) + add eval-only entrypoint (no retraining needed)

Browse files
Files changed (1) hide show
  1. fal_demo/train_modal.py +92 -26
fal_demo/train_modal.py CHANGED
@@ -2,8 +2,9 @@
2
  Future-as-Label Demo: GRPO Training on Modal
3
 
4
  Usage:
5
- uvx modal run --detach fal_demo/train_modal.py
6
- uvx modal run --detach fal_demo/train_modal.py --max-steps 400 --learning-rate 1e-6
 
7
 
8
  Prerequisites:
9
  - uvx modal setup
@@ -181,7 +182,7 @@ def train(max_steps:int=800, learning_rate:float=5e-6, num_generations:int=16,
181
  max_completion_length=max_completion_length,
182
  temperature=temperature, beta=0.0, scale_rewards=False,
183
  lr_scheduler_type="constant_with_warmup", warmup_ratio=0.05,
184
- save_steps=save_steps, logging_steps=10, fp16=True, report_to="none",
185
  seed=42, remove_unused_columns=False,
186
  push_to_hub=push_to_hub, hub_model_id=hub_model_id if push_to_hub else None)
187
 
@@ -243,32 +244,90 @@ def evaluate():
243
  if ap.exists(): set_peft_model_state_dict(model, safetensors.torch.load_file(str(ap)))
244
  tp = gen_preds(model, "tuned")
245
 
246
- outs = [r["outcome"] for r in eval_records]; tasks = [r["task"] for r in eval_records]
247
- def metrics(p,o):
248
- pa,oa=np.array(p),np.array(o,dtype=float); brier=float(np.mean((pa-oa)**2))
249
- bounds=np.linspace(0,1,11)
250
- ece=sum(m.sum()*abs(oa[m].mean()-pa[m].mean()) for i in range(10) if (m:=(pa>=bounds[i])&(pa<bounds[i+1])).sum()>0)/len(pa)
251
- br=float(oa.mean()); return {"brier":brier,"ece":float(ece),"base_rate":br,"brier_baseline":float(np.mean((br-oa)**2))}
252
-
253
- bm,tm=metrics(bp,outs),metrics(tp,outs)
254
- print(f"\n{'Metric':<12s} {'Base':>8s} {'Tuned':>8s} {'Ξ”':>8s}")
255
- print(f"{'Brier↓':<12s} {bm['brier']:>8.4f} {tm['brier']:>8.4f} {tm['brier']-bm['brier']:>+8.4f}")
256
- print(f"{'ECE↓':<12s} {bm['ece']:>8.4f} {tm['ece']:>8.4f} {tm['ece']-bm['ece']:>+8.4f}")
257
-
258
- tr={}
259
- for t in ["repurchase_90d","review_positive","delivery_ontime"]:
260
- idx=[i for i,x in enumerate(tasks) if x==t]
261
- tr[t]={"base":metrics([bp[i] for i in idx],[outs[i] for i in idx]),"tuned":metrics([tp[i] for i in idx],[outs[i] for i in idx])}
262
-
263
- bb=[(p-y)**2 for p,y in zip(bp,outs)]; tb=[(p-y)**2 for p,y in zip(tp,outs)]
264
- try: _,pv=wilcoxon(tb,bb,alternative='less'); print(f"Wilcoxon p={pv:.4f} {'βœ…' if pv<0.05 else '❌'}")
265
- except Exception as e: print(f"Wilcoxon: {e}"); pv=1.0
266
-
267
- res={"overall":{"base":bm,"tuned":tm},"per_task":tr,"wilcoxon_p":float(pv)}
268
- (CHECKPOINT_DIR/"fal-grpo"/"fal_results.json").write_text(json.dumps(res,indent=2))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
269
  checkpoint_vol.commit()
 
270
  return res
271
 
 
 
272
  @app.local_entrypoint()
273
  def main(max_steps:int=800, learning_rate:float=5e-6, num_generations:int=16,
274
  max_completion_length:int=48, run_eval:bool=True):
@@ -279,3 +338,10 @@ def main(max_steps:int=800, learning_rate:float=5e-6, num_generations:int=16,
279
  if run_eval:
280
  r = evaluate.remote()
281
  print(json.dumps(r, indent=2))
 
 
 
 
 
 
 
 
2
  Future-as-Label Demo: GRPO Training on Modal
3
 
4
  Usage:
5
+ uvx modal run fal_demo/train_modal.py # train + eval
6
+ uvx modal run fal_demo/train_modal.py::eval_only # eval only (reuse saved model)
7
+ uvx modal run --detach fal_demo/train_modal.py --num-generations 2 # custom args
8
 
9
  Prerequisites:
10
  - uvx modal setup
 
182
  max_completion_length=max_completion_length,
183
  temperature=temperature, beta=0.0, scale_rewards=False,
184
  lr_scheduler_type="constant_with_warmup", warmup_ratio=0.05,
185
+ save_steps=save_steps, logging_steps=10, fp16=False, report_to="none",
186
  seed=42, remove_unused_columns=False,
187
  push_to_hub=push_to_hub, hub_model_id=hub_model_id if push_to_hub else None)
188
 
 
244
  if ap.exists(): set_peft_model_state_dict(model, safetensors.torch.load_file(str(ap)))
245
  tp = gen_preds(model, "tuned")
246
 
247
+ outs = [r["outcome"] for r in eval_records]
248
+ tasks = [r["task"] for r in eval_records]
249
+
250
+ def metrics(p, o):
251
+ """Compute Brier and ECE. Handles empty arrays gracefully."""
252
+ pa, oa = np.array(p, dtype=float), np.array(o, dtype=float)
253
+ if len(pa) == 0:
254
+ return {"brier": 0.0, "ece": 0.0, "base_rate": 0.0, "brier_baseline": 0.0}
255
+ brier = float(np.mean((pa - oa) ** 2))
256
+ # ECE with 10 bins β€” skip empty bins
257
+ bounds = np.linspace(0, 1, 11)
258
+ ece_sum = 0.0
259
+ for i in range(10):
260
+ mask = (pa >= bounds[i]) & (pa < bounds[i + 1])
261
+ if mask.sum() > 0:
262
+ ece_sum += mask.sum() * abs(float(oa[mask].mean()) - float(pa[mask].mean()))
263
+ ece = ece_sum / len(pa)
264
+ br = float(oa.mean())
265
+ brier_baseline = float(np.mean((br - oa) ** 2))
266
+ return {"brier": brier, "ece": ece, "base_rate": br, "brier_baseline": brier_baseline}
267
+
268
+ # Overall metrics
269
+ bm, tm = metrics(bp, outs), metrics(tp, outs)
270
+ print(f"\n{'='*50}")
271
+ print(f"{'OVERALL RESULTS':^50}")
272
+ print(f"{'='*50}")
273
+ print(f"\n{'Metric':<20s} {'Base':>8s} {'Tuned':>8s} {'Ξ”':>8s}")
274
+ print(f"{'-'*46}")
275
+ print(f"{'Brier ↓':<20s} {bm['brier']:>8.4f} {tm['brier']:>8.4f} {tm['brier']-bm['brier']:>+8.4f}")
276
+ print(f"{'ECE ↓':<20s} {bm['ece']:>8.4f} {tm['ece']:>8.4f} {tm['ece']-bm['ece']:>+8.4f}")
277
+ print(f"{'Base rate Brier':<20s} {bm['brier_baseline']:>8.4f}")
278
+ print(f"{'Base rate':<20s} {bm['base_rate']:>8.4f}")
279
+
280
+ # Per-task metrics
281
+ print(f"\n{'='*50}")
282
+ print(f"{'PER-TASK BREAKDOWN':^50}")
283
+ print(f"{'='*50}")
284
+ print(f"\n{'Task':<20s} {'Base':>8s} {'Tuned':>8s} {'Ξ”':>8s} {'N':>5s}")
285
+ print(f"{'-'*50}")
286
+
287
+ tr = {}
288
+ for t in ["repurchase_90d", "review_positive", "delivery_ontime"]:
289
+ idx = [i for i, x in enumerate(tasks) if x == t]
290
+ if len(idx) == 0:
291
+ print(f"{t:<20s} {'N/A':>8s} {'N/A':>8s} {'N/A':>8s} {0:>5d}")
292
+ continue
293
+ t_bp = [bp[i] for i in idx]
294
+ t_tp = [tp[i] for i in idx]
295
+ t_outs = [outs[i] for i in idx]
296
+ tbm = metrics(t_bp, t_outs)
297
+ ttm = metrics(t_tp, t_outs)
298
+ delta = ttm["brier"] - tbm["brier"]
299
+ print(f"{t:<20s} {tbm['brier']:>8.4f} {ttm['brier']:>8.4f} {delta:>+8.4f} {len(idx):>5d}")
300
+ tr[t] = {"base": tbm, "tuned": ttm}
301
+
302
+ # Wilcoxon signed-rank test
303
+ print(f"\n{'='*50}")
304
+ bb = [(p - y) ** 2 for p, y in zip(bp, outs)]
305
+ tb = [(p - y) ** 2 for p, y in zip(tp, outs)]
306
+ try:
307
+ stat, pv = wilcoxon(tb, bb, alternative='less')
308
+ sig = "βœ… Tuned significantly better" if pv < 0.05 else "❌ Not significant"
309
+ print(f"Wilcoxon (tuned < base): W={stat:.0f}, p={pv:.4f} β†’ {sig}")
310
+ except Exception as e:
311
+ print(f"Wilcoxon: {e}")
312
+ pv = 1.0
313
+
314
+ # Save results
315
+ res = {
316
+ "overall": {"base": bm, "tuned": tm},
317
+ "per_task": tr,
318
+ "wilcoxon_p": float(pv),
319
+ "n_eval": len(eval_records),
320
+ "base_parse_rate": sum(1 for p in bp if p != 0.5) / len(bp),
321
+ "tuned_parse_rate": sum(1 for p in tp if p != 0.5) / len(tp),
322
+ }
323
+ results_path = CHECKPOINT_DIR / "fal-grpo" / "fal_results.json"
324
+ results_path.write_text(json.dumps(res, indent=2))
325
  checkpoint_vol.commit()
326
+ print(f"\nβœ“ Results saved to {results_path}")
327
  return res
328
 
329
+ # ── Entrypoints ───────────────────────────────────────────────────────────────
330
+
331
  @app.local_entrypoint()
332
  def main(max_steps:int=800, learning_rate:float=5e-6, num_generations:int=16,
333
  max_completion_length:int=48, run_eval:bool=True):
 
338
  if run_eval:
339
  r = evaluate.remote()
340
  print(json.dumps(r, indent=2))
341
+
342
+ @app.local_entrypoint()
343
+ def eval_only():
344
+ """Run evaluation only β€” uses model already saved in volume from previous training."""
345
+ print("Running evaluation only (model from volume)...")
346
+ r = evaluate.remote()
347
+ print(json.dumps(r, indent=2))