Sync InvoiceGuard code for GRPO training job
Browse files- training/train_grpo.py +141 -4
training/train_grpo.py
CHANGED
|
@@ -138,6 +138,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
|
|
| 138 |
from models import TaskID # type: ignore
|
| 139 |
from server.invoice_guard_environment import InvoiceGuardEnvironment # type: ignore
|
| 140 |
from tasks import HARD_TASK_LIST, TASK_LIST # type: ignore
|
|
|
|
| 141 |
from training.rollout import Trajectory, TrajectoryStep, rollout_episode # type: ignore
|
| 142 |
|
| 143 |
|
|
@@ -178,6 +179,12 @@ class TrainConfig:
|
|
| 178 |
max_new_tokens: int = 96
|
| 179 |
max_prompt_tokens: int = 2048
|
| 180 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
# LoRA
|
| 182 |
lora_r: int = 16
|
| 183 |
lora_alpha: int = 32
|
|
@@ -254,13 +261,129 @@ def compute_group_advantages(
|
|
| 254 |
) -> List[float]:
|
| 255 |
rewards = [trajectory_reward(t, grader_bonus) for t in trajectories]
|
| 256 |
if len(rewards) < 2:
|
| 257 |
-
return [0.0 for
|
| 258 |
mean = sum(rewards) / len(rewards)
|
| 259 |
var = sum((r - mean) ** 2 for r in rewards) / len(rewards)
|
| 260 |
-
|
|
|
|
|
|
|
| 261 |
return [(r - mean) / std for r in rewards]
|
| 262 |
|
| 263 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
# -----------------------------------------------------------------------------
|
| 265 |
# 6. Main training loop.
|
| 266 |
# -----------------------------------------------------------------------------
|
|
@@ -514,9 +637,17 @@ def train(cfg: TrainConfig) -> None:
|
|
| 514 |
fig.savefig(artifact_dir / "holdout_eval_curve.png", dpi=160)
|
| 515 |
plt.close(fig)
|
| 516 |
|
| 517 |
-
# -----
|
| 518 |
global_step = 0
|
| 519 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 520 |
evaluate("eval/init", global_step)
|
| 521 |
|
| 522 |
# ----- Training loop -------------------------------------------------------
|
|
@@ -717,6 +848,8 @@ def _parse_args() -> TrainConfig:
|
|
| 717 |
p.add_argument("--eval-holdout-hard", type=int, default=None)
|
| 718 |
p.add_argument("--max-new-tokens", type=int, default=None)
|
| 719 |
p.add_argument("--max-prompt-tokens", type=int, default=None)
|
|
|
|
|
|
|
| 720 |
args = p.parse_args()
|
| 721 |
|
| 722 |
cfg = TrainConfig()
|
|
@@ -740,6 +873,10 @@ def _parse_args() -> TrainConfig:
|
|
| 740 |
cfg.max_new_tokens = args.max_new_tokens
|
| 741 |
if args.max_prompt_tokens is not None:
|
| 742 |
cfg.max_prompt_tokens = args.max_prompt_tokens
|
|
|
|
|
|
|
|
|
|
|
|
|
| 743 |
if args.no_push:
|
| 744 |
cfg.push_to_hub = False
|
| 745 |
if args.no_4bit:
|
|
|
|
| 138 |
from models import TaskID # type: ignore
|
| 139 |
from server.invoice_guard_environment import InvoiceGuardEnvironment # type: ignore
|
| 140 |
from tasks import HARD_TASK_LIST, TASK_LIST # type: ignore
|
| 141 |
+
from inference import SYSTEM_PROMPT, build_action, build_observation_prompt # type: ignore
|
| 142 |
from training.rollout import Trajectory, TrajectoryStep, rollout_episode # type: ignore
|
| 143 |
|
| 144 |
|
|
|
|
| 179 |
max_new_tokens: int = 96
|
| 180 |
max_prompt_tokens: int = 2048
|
| 181 |
|
| 182 |
+
# Tiny behavior warm-start. Smoke showed the raw model sometimes echoes the
|
| 183 |
+
# observation instead of emitting JSON; this teaches format before RL.
|
| 184 |
+
format_warmup: bool = True
|
| 185 |
+
format_warmup_tasks: int = 8
|
| 186 |
+
format_warmup_lr: float = 5e-5
|
| 187 |
+
|
| 188 |
# LoRA
|
| 189 |
lora_r: int = 16
|
| 190 |
lora_alpha: int = 32
|
|
|
|
| 261 |
) -> List[float]:
|
| 262 |
rewards = [trajectory_reward(t, grader_bonus) for t in trajectories]
|
| 263 |
if len(rewards) < 2:
|
| 264 |
+
return [max(min(r, 2.0), -2.0) for r in rewards]
|
| 265 |
mean = sum(rewards) / len(rewards)
|
| 266 |
var = sum((r - mean) ** 2 for r in rewards) / len(rewards)
|
| 267 |
+
if var < 1e-10:
|
| 268 |
+
return [max(min(r, 2.0), -2.0) for r in rewards]
|
| 269 |
+
std = var ** 0.5
|
| 270 |
return [(r - mean) / std for r in rewards]
|
| 271 |
|
| 272 |
|
| 273 |
+
def _format_warmup_actions(env: InvoiceGuardEnvironment, task_id: TaskID) -> list[dict]:
|
| 274 |
+
case = getattr(env, "_case", None)
|
| 275 |
+
if case is None:
|
| 276 |
+
env.reset(task_id=task_id.value)
|
| 277 |
+
case = getattr(env, "_case", None)
|
| 278 |
+
assert case is not None
|
| 279 |
+
gt = case.ground_truth
|
| 280 |
+
evidence = list(dict.fromkeys([
|
| 281 |
+
"inspect_purchase_order",
|
| 282 |
+
"inspect_goods_receipt_note",
|
| 283 |
+
"inspect_invoice_line_items",
|
| 284 |
+
"inspect_vendor_profile",
|
| 285 |
+
"compare_quantity",
|
| 286 |
+
"compare_price",
|
| 287 |
+
"compare_totals",
|
| 288 |
+
"check_for_duplicate_invoice",
|
| 289 |
+
"inspect_policy_rules",
|
| 290 |
+
*gt.acceptable_evidence,
|
| 291 |
+
]))
|
| 292 |
+
explanation = "Key findings: " + "; ".join(gt.key_findings[:3])
|
| 293 |
+
return [
|
| 294 |
+
{"action_type": "inspect_purchase_order"},
|
| 295 |
+
{"action_type": "inspect_goods_receipt_note"},
|
| 296 |
+
{"action_type": "inspect_invoice_line_items"},
|
| 297 |
+
{"action_type": "inspect_vendor_profile"},
|
| 298 |
+
{"action_type": "compare_quantity"},
|
| 299 |
+
{"action_type": "compare_price"},
|
| 300 |
+
{"action_type": "compare_totals"},
|
| 301 |
+
{"action_type": "check_for_duplicate_invoice"},
|
| 302 |
+
{"action_type": "inspect_policy_rules"},
|
| 303 |
+
{
|
| 304 |
+
"action_type": "submit_final_resolution",
|
| 305 |
+
"final_decision": gt.correct_decision.value,
|
| 306 |
+
"exception_type": gt.correct_exception_type.value,
|
| 307 |
+
"evidence_references": evidence,
|
| 308 |
+
"explanation": explanation,
|
| 309 |
+
"confidence": 0.9,
|
| 310 |
+
},
|
| 311 |
+
]
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def run_format_warmup(
|
| 315 |
+
policy,
|
| 316 |
+
tokenizer,
|
| 317 |
+
optimizer,
|
| 318 |
+
env: InvoiceGuardEnvironment,
|
| 319 |
+
tasks: list[TaskID],
|
| 320 |
+
cfg: TrainConfig,
|
| 321 |
+
device: torch.device,
|
| 322 |
+
) -> dict:
|
| 323 |
+
if not cfg.format_warmup or not tasks:
|
| 324 |
+
return {"format_warmup/enabled": 0.0, "format_warmup/n_pairs": 0.0}
|
| 325 |
+
|
| 326 |
+
old_lrs = [group["lr"] for group in optimizer.param_groups]
|
| 327 |
+
for group in optimizer.param_groups:
|
| 328 |
+
group["lr"] = cfg.format_warmup_lr
|
| 329 |
+
|
| 330 |
+
policy.train()
|
| 331 |
+
n_pairs = 0
|
| 332 |
+
total_loss = 0.0
|
| 333 |
+
for task_id in tasks[: cfg.format_warmup_tasks]:
|
| 334 |
+
obs = env.reset(task_id=task_id.value)
|
| 335 |
+
messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}]
|
| 336 |
+
for action_dict in _format_warmup_actions(env, task_id):
|
| 337 |
+
user_msg = build_observation_prompt(obs, is_first=(len(messages) == 1))
|
| 338 |
+
messages.append({"role": "user", "content": user_msg})
|
| 339 |
+
prompt_text = tokenizer.apply_chat_template(
|
| 340 |
+
messages,
|
| 341 |
+
tokenize=False,
|
| 342 |
+
add_generation_prompt=True,
|
| 343 |
+
)
|
| 344 |
+
completion_text = json.dumps(action_dict, ensure_ascii=False)
|
| 345 |
+
prompt_ids = tokenizer(
|
| 346 |
+
prompt_text,
|
| 347 |
+
return_tensors="pt",
|
| 348 |
+
add_special_tokens=False,
|
| 349 |
+
truncation=True,
|
| 350 |
+
max_length=cfg.max_prompt_tokens,
|
| 351 |
+
).input_ids[0]
|
| 352 |
+
completion_ids = tokenizer(
|
| 353 |
+
completion_text,
|
| 354 |
+
return_tensors="pt",
|
| 355 |
+
add_special_tokens=False,
|
| 356 |
+
).input_ids[0]
|
| 357 |
+
lp = _completion_logprobs(policy, prompt_ids, completion_ids, device)
|
| 358 |
+
loss = -lp / max(int(completion_ids.shape[0]), 1)
|
| 359 |
+
loss.backward()
|
| 360 |
+
total_loss += float(loss.detach().item())
|
| 361 |
+
n_pairs += 1
|
| 362 |
+
|
| 363 |
+
messages.append({"role": "assistant", "content": completion_text})
|
| 364 |
+
obs = env.step(build_action(action_dict))
|
| 365 |
+
if obs.done:
|
| 366 |
+
break
|
| 367 |
+
|
| 368 |
+
if n_pairs:
|
| 369 |
+
torch.nn.utils.clip_grad_norm_(
|
| 370 |
+
[p for p in policy.parameters() if p.requires_grad],
|
| 371 |
+
cfg.grad_clip,
|
| 372 |
+
)
|
| 373 |
+
optimizer.step()
|
| 374 |
+
optimizer.zero_grad(set_to_none=True)
|
| 375 |
+
|
| 376 |
+
for group, lr in zip(optimizer.param_groups, old_lrs):
|
| 377 |
+
group["lr"] = lr
|
| 378 |
+
|
| 379 |
+
return {
|
| 380 |
+
"format_warmup/enabled": 1.0,
|
| 381 |
+
"format_warmup/n_pairs": float(n_pairs),
|
| 382 |
+
"format_warmup/loss": total_loss / max(n_pairs, 1),
|
| 383 |
+
"format_warmup/n_tasks": float(min(len(tasks), cfg.format_warmup_tasks)),
|
| 384 |
+
}
|
| 385 |
+
|
| 386 |
+
|
| 387 |
# -----------------------------------------------------------------------------
|
| 388 |
# 6. Main training loop.
|
| 389 |
# -----------------------------------------------------------------------------
|
|
|
|
| 637 |
fig.savefig(artifact_dir / "holdout_eval_curve.png", dpi=160)
|
| 638 |
plt.close(fig)
|
| 639 |
|
| 640 |
+
# ----- Format warm-start ---------------------------------------------------
|
| 641 |
global_step = 0
|
| 642 |
+
if cfg.format_warmup:
|
| 643 |
+
print("\n=== format warm-start (JSON action behavior) ===", flush=True)
|
| 644 |
+
warmup_metrics = run_format_warmup(
|
| 645 |
+
policy, tokenizer, optimizer, env, train_tasks, cfg, device
|
| 646 |
+
)
|
| 647 |
+
_log(warmup_metrics, global_step)
|
| 648 |
+
|
| 649 |
+
# ----- Initial eval --------------------------------------------------------
|
| 650 |
+
print("\n=== initial eval (after format warm-start) ===", flush=True)
|
| 651 |
evaluate("eval/init", global_step)
|
| 652 |
|
| 653 |
# ----- Training loop -------------------------------------------------------
|
|
|
|
| 848 |
p.add_argument("--eval-holdout-hard", type=int, default=None)
|
| 849 |
p.add_argument("--max-new-tokens", type=int, default=None)
|
| 850 |
p.add_argument("--max-prompt-tokens", type=int, default=None)
|
| 851 |
+
p.add_argument("--no-format-warmup", action="store_true")
|
| 852 |
+
p.add_argument("--format-warmup-tasks", type=int, default=None)
|
| 853 |
args = p.parse_args()
|
| 854 |
|
| 855 |
cfg = TrainConfig()
|
|
|
|
| 873 |
cfg.max_new_tokens = args.max_new_tokens
|
| 874 |
if args.max_prompt_tokens is not None:
|
| 875 |
cfg.max_prompt_tokens = args.max_prompt_tokens
|
| 876 |
+
if args.no_format_warmup:
|
| 877 |
+
cfg.format_warmup = False
|
| 878 |
+
if args.format_warmup_tasks is not None:
|
| 879 |
+
cfg.format_warmup_tasks = args.format_warmup_tasks
|
| 880 |
if args.no_push:
|
| 881 |
cfg.push_to_hub = False
|
| 882 |
if args.no_4bit:
|