renpas22 commited on
Commit
5419afd
·
1 Parent(s): 5cfd8d6

Add checkpoint resumption - automatically resume from latest checkpoint

Browse files
Files changed (1) hide show
  1. src/reasoning/step_level_cot.py +75 -1
src/reasoning/step_level_cot.py CHANGED
@@ -486,6 +486,17 @@ class StepLevelCoTTrainer:
486
  weight_decay=getattr(self.config, 'weight_decay', 0.01),
487
  )
488
 
 
 
 
 
 
 
 
 
 
 
 
489
  # Prepare with accelerator
490
  self.model, optimizer, train_dataloader = self.accelerator.prepare(
491
  self.model, optimizer, train_dataloader
@@ -493,12 +504,13 @@ class StepLevelCoTTrainer:
493
 
494
  # Training loop
495
  self.model.train()
496
- global_step = 0
497
  total_loss = 0
498
 
499
  from tqdm import tqdm
500
  progress_bar = tqdm(
501
  total=max_steps,
 
502
  desc="SFT Training",
503
  disable=not self.accelerator.is_local_main_process,
504
  )
@@ -865,6 +877,68 @@ class StepLevelCoTTrainer:
865
 
866
  logger.info(f"Checkpoint saved to {save_path}")
867
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
868
  def _save_prm(self, epoch: int) -> None:
869
  """Save PRM checkpoint."""
870
  save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"
 
486
  weight_decay=getattr(self.config, 'weight_decay', 0.01),
487
  )
488
 
489
+ # Check for existing checkpoint to resume from
490
+ resume_step = 0
491
+ checkpoint_path = self._find_latest_checkpoint("sft")
492
+ if checkpoint_path:
493
+ try:
494
+ resume_step = self._load_checkpoint(checkpoint_path, "sft")
495
+ logger.info(f"Resuming SFT training from step {resume_step}")
496
+ except Exception as e:
497
+ logger.warning(f"Failed to load checkpoint: {e}. Starting from scratch.")
498
+ resume_step = 0
499
+
500
  # Prepare with accelerator
501
  self.model, optimizer, train_dataloader = self.accelerator.prepare(
502
  self.model, optimizer, train_dataloader
 
504
 
505
  # Training loop
506
  self.model.train()
507
+ global_step = resume_step
508
  total_loss = 0
509
 
510
  from tqdm import tqdm
511
  progress_bar = tqdm(
512
  total=max_steps,
513
+ initial=resume_step,
514
  desc="SFT Training",
515
  disable=not self.accelerator.is_local_main_process,
516
  )
 
877
 
878
  logger.info(f"Checkpoint saved to {save_path}")
879
 
880
+ def _find_latest_checkpoint(self, phase: str) -> Optional[Path]:
881
+ """Find the latest checkpoint for a specific phase."""
882
+ checkpoint_dir = Path(self.config.output_dir)
883
+ if not checkpoint_dir.exists():
884
+ return None
885
+
886
+ # Find all checkpoints for this phase
887
+ checkpoints = list(checkpoint_dir.glob(f"checkpoint-{phase}-*"))
888
+ if not checkpoints:
889
+ return None
890
+
891
+ # Extract step numbers and find the latest
892
+ checkpoint_steps = []
893
+ for ckpt in checkpoints:
894
+ try:
895
+ step = int(ckpt.name.split("-")[-1])
896
+ checkpoint_steps.append((step, ckpt))
897
+ except ValueError:
898
+ continue
899
+
900
+ if not checkpoint_steps:
901
+ return None
902
+
903
+ # Return checkpoint with highest step number
904
+ latest_step, latest_path = max(checkpoint_steps, key=lambda x: x[0])
905
+ logger.info(f"Found checkpoint at step {latest_step}: {latest_path}")
906
+ return latest_path
907
+
908
+ def _load_checkpoint(self, checkpoint_path: Path, phase: str) -> int:
909
+ """Load checkpoint and return the step number."""
910
+ if phase == "sft":
911
+ # Load model from checkpoint
912
+ from transformers import AutoModelForVision2Seq
913
+ logger.info(f"Loading SFT checkpoint from {checkpoint_path}")
914
+
915
+ # Load the checkpoint model
916
+ checkpoint_model = AutoModelForVision2Seq.from_pretrained(
917
+ checkpoint_path,
918
+ torch_dtype=torch.bfloat16,
919
+ device_map=self.accelerator.device,
920
+ )
921
+
922
+ # Copy weights to current model
923
+ self.model.load_state_dict(checkpoint_model.state_dict())
924
+ del checkpoint_model
925
+
926
+ elif phase == "prm":
927
+ logger.info(f"Loading PRM checkpoint from {checkpoint_path}")
928
+ self.prm.load_state_dict(
929
+ torch.load(checkpoint_path / "prm_model.pt", map_location=self.accelerator.device)
930
+ )
931
+ elif phase == "rl":
932
+ logger.info(f"Loading RL checkpoint from {checkpoint_path}")
933
+ self.rl_trainer.policy.load_state_dict(
934
+ torch.load(checkpoint_path / "rl_policy.pt", map_location=self.accelerator.device)
935
+ )
936
+
937
+ # Extract step number from checkpoint path
938
+ step = int(checkpoint_path.name.split("-")[-1])
939
+ logger.info(f"Resumed from checkpoint at step {step}")
940
+ return step
941
+
942
  def _save_prm(self, epoch: int) -> None:
943
  """Save PRM checkpoint."""
944
  save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"