piyush-mk commited on
Commit
7db1128
·
verified ·
1 Parent(s): 7c278c9

Sync InvoiceGuard code for GRPO training job

Browse files
Files changed (1) hide show
  1. training/train_grpo.py +141 -4
training/train_grpo.py CHANGED
@@ -138,6 +138,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
138
  from models import TaskID # type: ignore
139
  from server.invoice_guard_environment import InvoiceGuardEnvironment # type: ignore
140
  from tasks import HARD_TASK_LIST, TASK_LIST # type: ignore
 
141
  from training.rollout import Trajectory, TrajectoryStep, rollout_episode # type: ignore
142
 
143
 
@@ -178,6 +179,12 @@ class TrainConfig:
178
  max_new_tokens: int = 96
179
  max_prompt_tokens: int = 2048
180
 
 
 
 
 
 
 
181
  # LoRA
182
  lora_r: int = 16
183
  lora_alpha: int = 32
@@ -254,13 +261,129 @@ def compute_group_advantages(
254
  ) -> List[float]:
255
  rewards = [trajectory_reward(t, grader_bonus) for t in trajectories]
256
  if len(rewards) < 2:
257
- return [0.0 for _ in rewards]
258
  mean = sum(rewards) / len(rewards)
259
  var = sum((r - mean) ** 2 for r in rewards) / len(rewards)
260
- std = max(var ** 0.5, 1e-6)
 
 
261
  return [(r - mean) / std for r in rewards]
262
 
263
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
264
  # -----------------------------------------------------------------------------
265
  # 6. Main training loop.
266
  # -----------------------------------------------------------------------------
@@ -514,9 +637,17 @@ def train(cfg: TrainConfig) -> None:
514
  fig.savefig(artifact_dir / "holdout_eval_curve.png", dpi=160)
515
  plt.close(fig)
516
 
517
- # ----- Initial eval --------------------------------------------------------
518
  global_step = 0
519
- print("\n=== initial eval (LoRA initialised but untrained) ===", flush=True)
 
 
 
 
 
 
 
 
520
  evaluate("eval/init", global_step)
521
 
522
  # ----- Training loop -------------------------------------------------------
@@ -717,6 +848,8 @@ def _parse_args() -> TrainConfig:
717
  p.add_argument("--eval-holdout-hard", type=int, default=None)
718
  p.add_argument("--max-new-tokens", type=int, default=None)
719
  p.add_argument("--max-prompt-tokens", type=int, default=None)
 
 
720
  args = p.parse_args()
721
 
722
  cfg = TrainConfig()
@@ -740,6 +873,10 @@ def _parse_args() -> TrainConfig:
740
  cfg.max_new_tokens = args.max_new_tokens
741
  if args.max_prompt_tokens is not None:
742
  cfg.max_prompt_tokens = args.max_prompt_tokens
 
 
 
 
743
  if args.no_push:
744
  cfg.push_to_hub = False
745
  if args.no_4bit:
 
138
  from models import TaskID # type: ignore
139
  from server.invoice_guard_environment import InvoiceGuardEnvironment # type: ignore
140
  from tasks import HARD_TASK_LIST, TASK_LIST # type: ignore
141
+ from inference import SYSTEM_PROMPT, build_action, build_observation_prompt # type: ignore
142
  from training.rollout import Trajectory, TrajectoryStep, rollout_episode # type: ignore
143
 
144
 
 
179
  max_new_tokens: int = 96
180
  max_prompt_tokens: int = 2048
181
 
182
+ # Tiny behavior warm-start. Smoke showed the raw model sometimes echoes the
183
+ # observation instead of emitting JSON; this teaches format before RL.
184
+ format_warmup: bool = True
185
+ format_warmup_tasks: int = 8
186
+ format_warmup_lr: float = 5e-5
187
+
188
  # LoRA
189
  lora_r: int = 16
190
  lora_alpha: int = 32
 
261
  ) -> List[float]:
262
  rewards = [trajectory_reward(t, grader_bonus) for t in trajectories]
263
  if len(rewards) < 2:
264
+ return [max(min(r, 2.0), -2.0) for r in rewards]
265
  mean = sum(rewards) / len(rewards)
266
  var = sum((r - mean) ** 2 for r in rewards) / len(rewards)
267
+ if var < 1e-10:
268
+ return [max(min(r, 2.0), -2.0) for r in rewards]
269
+ std = var ** 0.5
270
  return [(r - mean) / std for r in rewards]
271
 
272
 
273
+ def _format_warmup_actions(env: InvoiceGuardEnvironment, task_id: TaskID) -> list[dict]:
274
+ case = getattr(env, "_case", None)
275
+ if case is None:
276
+ env.reset(task_id=task_id.value)
277
+ case = getattr(env, "_case", None)
278
+ assert case is not None
279
+ gt = case.ground_truth
280
+ evidence = list(dict.fromkeys([
281
+ "inspect_purchase_order",
282
+ "inspect_goods_receipt_note",
283
+ "inspect_invoice_line_items",
284
+ "inspect_vendor_profile",
285
+ "compare_quantity",
286
+ "compare_price",
287
+ "compare_totals",
288
+ "check_for_duplicate_invoice",
289
+ "inspect_policy_rules",
290
+ *gt.acceptable_evidence,
291
+ ]))
292
+ explanation = "Key findings: " + "; ".join(gt.key_findings[:3])
293
+ return [
294
+ {"action_type": "inspect_purchase_order"},
295
+ {"action_type": "inspect_goods_receipt_note"},
296
+ {"action_type": "inspect_invoice_line_items"},
297
+ {"action_type": "inspect_vendor_profile"},
298
+ {"action_type": "compare_quantity"},
299
+ {"action_type": "compare_price"},
300
+ {"action_type": "compare_totals"},
301
+ {"action_type": "check_for_duplicate_invoice"},
302
+ {"action_type": "inspect_policy_rules"},
303
+ {
304
+ "action_type": "submit_final_resolution",
305
+ "final_decision": gt.correct_decision.value,
306
+ "exception_type": gt.correct_exception_type.value,
307
+ "evidence_references": evidence,
308
+ "explanation": explanation,
309
+ "confidence": 0.9,
310
+ },
311
+ ]
312
+
313
+
314
+ def run_format_warmup(
315
+ policy,
316
+ tokenizer,
317
+ optimizer,
318
+ env: InvoiceGuardEnvironment,
319
+ tasks: list[TaskID],
320
+ cfg: TrainConfig,
321
+ device: torch.device,
322
+ ) -> dict:
323
+ if not cfg.format_warmup or not tasks:
324
+ return {"format_warmup/enabled": 0.0, "format_warmup/n_pairs": 0.0}
325
+
326
+ old_lrs = [group["lr"] for group in optimizer.param_groups]
327
+ for group in optimizer.param_groups:
328
+ group["lr"] = cfg.format_warmup_lr
329
+
330
+ policy.train()
331
+ n_pairs = 0
332
+ total_loss = 0.0
333
+ for task_id in tasks[: cfg.format_warmup_tasks]:
334
+ obs = env.reset(task_id=task_id.value)
335
+ messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}]
336
+ for action_dict in _format_warmup_actions(env, task_id):
337
+ user_msg = build_observation_prompt(obs, is_first=(len(messages) == 1))
338
+ messages.append({"role": "user", "content": user_msg})
339
+ prompt_text = tokenizer.apply_chat_template(
340
+ messages,
341
+ tokenize=False,
342
+ add_generation_prompt=True,
343
+ )
344
+ completion_text = json.dumps(action_dict, ensure_ascii=False)
345
+ prompt_ids = tokenizer(
346
+ prompt_text,
347
+ return_tensors="pt",
348
+ add_special_tokens=False,
349
+ truncation=True,
350
+ max_length=cfg.max_prompt_tokens,
351
+ ).input_ids[0]
352
+ completion_ids = tokenizer(
353
+ completion_text,
354
+ return_tensors="pt",
355
+ add_special_tokens=False,
356
+ ).input_ids[0]
357
+ lp = _completion_logprobs(policy, prompt_ids, completion_ids, device)
358
+ loss = -lp / max(int(completion_ids.shape[0]), 1)
359
+ loss.backward()
360
+ total_loss += float(loss.detach().item())
361
+ n_pairs += 1
362
+
363
+ messages.append({"role": "assistant", "content": completion_text})
364
+ obs = env.step(build_action(action_dict))
365
+ if obs.done:
366
+ break
367
+
368
+ if n_pairs:
369
+ torch.nn.utils.clip_grad_norm_(
370
+ [p for p in policy.parameters() if p.requires_grad],
371
+ cfg.grad_clip,
372
+ )
373
+ optimizer.step()
374
+ optimizer.zero_grad(set_to_none=True)
375
+
376
+ for group, lr in zip(optimizer.param_groups, old_lrs):
377
+ group["lr"] = lr
378
+
379
+ return {
380
+ "format_warmup/enabled": 1.0,
381
+ "format_warmup/n_pairs": float(n_pairs),
382
+ "format_warmup/loss": total_loss / max(n_pairs, 1),
383
+ "format_warmup/n_tasks": float(min(len(tasks), cfg.format_warmup_tasks)),
384
+ }
385
+
386
+
387
  # -----------------------------------------------------------------------------
388
  # 6. Main training loop.
389
  # -----------------------------------------------------------------------------
 
637
  fig.savefig(artifact_dir / "holdout_eval_curve.png", dpi=160)
638
  plt.close(fig)
639
 
640
+ # ----- Format warm-start ---------------------------------------------------
641
  global_step = 0
642
+ if cfg.format_warmup:
643
+ print("\n=== format warm-start (JSON action behavior) ===", flush=True)
644
+ warmup_metrics = run_format_warmup(
645
+ policy, tokenizer, optimizer, env, train_tasks, cfg, device
646
+ )
647
+ _log(warmup_metrics, global_step)
648
+
649
+ # ----- Initial eval --------------------------------------------------------
650
+ print("\n=== initial eval (after format warm-start) ===", flush=True)
651
  evaluate("eval/init", global_step)
652
 
653
  # ----- Training loop -------------------------------------------------------
 
848
  p.add_argument("--eval-holdout-hard", type=int, default=None)
849
  p.add_argument("--max-new-tokens", type=int, default=None)
850
  p.add_argument("--max-prompt-tokens", type=int, default=None)
851
+ p.add_argument("--no-format-warmup", action="store_true")
852
+ p.add_argument("--format-warmup-tasks", type=int, default=None)
853
  args = p.parse_args()
854
 
855
  cfg = TrainConfig()
 
873
  cfg.max_new_tokens = args.max_new_tokens
874
  if args.max_prompt_tokens is not None:
875
  cfg.max_prompt_tokens = args.max_prompt_tokens
876
+ if args.no_format_warmup:
877
+ cfg.format_warmup = False
878
+ if args.format_warmup_tasks is not None:
879
+ cfg.format_warmup_tasks = args.format_warmup_tasks
880
  if args.no_push:
881
  cfg.push_to_hub = False
882
  if args.no_4bit: