renpas22 commited on
Commit
41bcc92
·
1 Parent(s): 85ab8c2

Fix train_prm and train_rl signatures to accept max_steps and learning_rate

Browse files
Files changed (1) hide show
  1. src/reasoning/step_level_cot.py +63 -37
src/reasoning/step_level_cot.py CHANGED
@@ -349,23 +349,30 @@ class StepLevelCoTTrainer:
349
 
350
  def train_prm(
351
  self,
352
- train_dataset: StepDataset,
353
- val_dataset: Optional[StepDataset] = None,
354
- num_epochs: int = 10,
 
355
  ) -> None:
356
  """
357
  Train Process Reward Model.
358
 
359
  Args:
360
- train_dataset: Training dataset with step-level rewards
361
- val_dataset: Optional validation dataset
362
- num_epochs: Number of training epochs
 
363
  """
364
  logger.info("Starting PRM training")
 
 
 
 
 
365
 
366
  prm_trainer = PRMTrainer(
367
  model=self.prm,
368
- learning_rate=self.config.prm_learning_rate,
369
  weight_decay=self.config.weight_decay,
370
  warmup_steps=self.config.warmup_steps,
371
  )
@@ -383,19 +390,22 @@ class StepLevelCoTTrainer:
383
  )
384
 
385
  best_val_loss = float('inf')
 
386
 
387
- for epoch in range(num_epochs):
388
- # Training loop
389
- prm_trainer.model.train()
390
- total_loss = 0
391
-
392
- progress_bar = tqdm(
393
- train_dataloader,
394
- desc=f"PRM Epoch {epoch + 1}/{num_epochs}",
395
- disable=not self.accelerator.is_local_main_process,
396
- )
397
-
398
- for batch in progress_bar:
 
 
399
  # Extract batch data
400
  vision_features = batch['visual_features']
401
  step_embeddings = batch['step_descriptions'] # Need to encode these
@@ -410,37 +420,53 @@ class StepLevelCoTTrainer:
410
  target_rewards,
411
  )
412
 
413
- total_loss += metrics['loss']
414
- progress_bar.set_postfix({'loss': metrics['loss']})
415
-
416
- avg_loss = total_loss / len(train_dataloader)
417
- logger.info(f"Epoch {epoch + 1} - Avg Loss: {avg_loss:.4f}")
418
-
419
- # Validation
420
- if val_dataset is not None:
421
- val_metrics = self._evaluate_prm(prm_trainer, val_dataset)
422
- logger.info(f"Validation: {val_metrics}")
423
 
424
- # Save best model
425
- if val_metrics['mse'] < best_val_loss:
426
- best_val_loss = val_metrics['mse']
427
- self._save_prm(epoch)
 
 
 
 
 
 
 
 
 
 
 
 
428
 
429
  logger.info("PRM training completed")
430
 
431
  def train_rl(
432
  self,
433
- train_dataset: StepDataset,
434
- num_iterations: int = 1000,
 
 
435
  ) -> None:
436
  """
437
  Train policy with reinforcement learning.
438
 
439
  Args:
440
- train_dataset: Training dataset
441
- num_iterations: Number of RL iterations
 
 
442
  """
443
  logger.info("Starting RL training")
 
 
 
 
 
 
 
444
 
445
  self.rl_trainer.train(
446
  train_dataset=train_dataset,
 
349
 
350
  def train_prm(
351
  self,
352
+ max_steps: int = 5000,
353
+ learning_rate: float = 1e-5,
354
+ save_steps: int = 500,
355
+ eval_steps: int = 500,
356
  ) -> None:
357
  """
358
  Train Process Reward Model.
359
 
360
  Args:
361
+ max_steps: Maximum training steps
362
+ learning_rate: Learning rate for PRM training
363
+ save_steps: Save checkpoint every N steps
364
+ eval_steps: Evaluate every N steps
365
  """
366
  logger.info("Starting PRM training")
367
+ logger.info(f"Max steps: {max_steps}, LR: {learning_rate}")
368
+
369
+ # Load datasets using the trainer's dataset loading methods
370
+ train_dataset = self.load_step_dataset(split='train')
371
+ val_dataset = self.load_step_dataset(split='validation') if eval_steps > 0 else None
372
 
373
  prm_trainer = PRMTrainer(
374
  model=self.prm,
375
+ learning_rate=learning_rate, # Use the passed parameter
376
  weight_decay=self.config.weight_decay,
377
  warmup_steps=self.config.warmup_steps,
378
  )
 
390
  )
391
 
392
  best_val_loss = float('inf')
393
+ global_step = 0
394
 
395
+ # Training loop using steps instead of epochs
396
+ prm_trainer.model.train()
397
+
398
+ progress_bar = tqdm(
399
+ total=max_steps,
400
+ desc=f"PRM Training",
401
+ disable=not self.accelerator.is_local_main_process,
402
+ )
403
+
404
+ while global_step < max_steps:
405
+ for batch in train_dataloader:
406
+ if global_step >= max_steps:
407
+ break
408
+
409
  # Extract batch data
410
  vision_features = batch['visual_features']
411
  step_embeddings = batch['step_descriptions'] # Need to encode these
 
420
  target_rewards,
421
  )
422
 
423
+ global_step += 1
424
+ progress_bar.update(1)
425
+ progress_bar.set_postfix({'loss': metrics['loss'], 'step': global_step})
 
 
 
 
 
 
 
426
 
427
+ # Save checkpoint
428
+ if global_step % save_steps == 0:
429
+ logger.info(f"Saving checkpoint at step {global_step}")
430
+ self._save_prm(global_step)
431
+
432
+ # Validation
433
+ if eval_steps > 0 and global_step % eval_steps == 0 and val_dataset is not None:
434
+ val_metrics = self._evaluate_prm(prm_trainer, val_dataset)
435
+ logger.info(f"Step {global_step} - Validation: {val_metrics}")
436
+
437
+ # Save best model
438
+ if val_metrics['mse'] < best_val_loss:
439
+ best_val_loss = val_metrics['mse']
440
+ self._save_prm(f"best_step_{global_step}")
441
+
442
+ progress_bar.close()
443
 
444
  logger.info("PRM training completed")
445
 
446
  def train_rl(
447
  self,
448
+ max_steps: int = 8000,
449
+ learning_rate: float = 5e-6,
450
+ save_steps: int = 500,
451
+ eval_steps: int = 500,
452
  ) -> None:
453
  """
454
  Train policy with reinforcement learning.
455
 
456
  Args:
457
+ max_steps: Maximum training steps
458
+ learning_rate: Learning rate for RL training
459
+ save_steps: Save checkpoint every N steps
460
+ eval_steps: Evaluate every N steps
461
  """
462
  logger.info("Starting RL training")
463
+ logger.info(f"Max steps: {max_steps}, LR: {learning_rate}")
464
+
465
+ # Load dataset using the trainer's dataset loading methods
466
+ train_dataset = self.load_step_dataset(split='train')
467
+
468
+ # Convert max_steps to iterations (approximate)
469
+ num_iterations = max_steps // self.config.train_batch_size
470
 
471
  self.rl_trainer.train(
472
  train_dataset=train_dataset,