renpas22 commited on
Commit ·
5419afd
1
Parent(s): 5cfd8d6
Add checkpoint resumption - automatically resume from latest checkpoint
Browse files
src/reasoning/step_level_cot.py
CHANGED
|
@@ -486,6 +486,17 @@ class StepLevelCoTTrainer:
|
|
| 486 |
weight_decay=getattr(self.config, 'weight_decay', 0.01),
|
| 487 |
)
|
| 488 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
# Prepare with accelerator
|
| 490 |
self.model, optimizer, train_dataloader = self.accelerator.prepare(
|
| 491 |
self.model, optimizer, train_dataloader
|
|
@@ -493,12 +504,13 @@ class StepLevelCoTTrainer:
|
|
| 493 |
|
| 494 |
# Training loop
|
| 495 |
self.model.train()
|
| 496 |
-
global_step =
|
| 497 |
total_loss = 0
|
| 498 |
|
| 499 |
from tqdm import tqdm
|
| 500 |
progress_bar = tqdm(
|
| 501 |
total=max_steps,
|
|
|
|
| 502 |
desc="SFT Training",
|
| 503 |
disable=not self.accelerator.is_local_main_process,
|
| 504 |
)
|
|
@@ -865,6 +877,68 @@ class StepLevelCoTTrainer:
|
|
| 865 |
|
| 866 |
logger.info(f"Checkpoint saved to {save_path}")
|
| 867 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 868 |
def _save_prm(self, epoch: int) -> None:
|
| 869 |
"""Save PRM checkpoint."""
|
| 870 |
save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"
|
|
|
|
| 486 |
weight_decay=getattr(self.config, 'weight_decay', 0.01),
|
| 487 |
)
|
| 488 |
|
| 489 |
+
# Check for existing checkpoint to resume from
|
| 490 |
+
resume_step = 0
|
| 491 |
+
checkpoint_path = self._find_latest_checkpoint("sft")
|
| 492 |
+
if checkpoint_path:
|
| 493 |
+
try:
|
| 494 |
+
resume_step = self._load_checkpoint(checkpoint_path, "sft")
|
| 495 |
+
logger.info(f"Resuming SFT training from step {resume_step}")
|
| 496 |
+
except Exception as e:
|
| 497 |
+
logger.warning(f"Failed to load checkpoint: {e}. Starting from scratch.")
|
| 498 |
+
resume_step = 0
|
| 499 |
+
|
| 500 |
# Prepare with accelerator
|
| 501 |
self.model, optimizer, train_dataloader = self.accelerator.prepare(
|
| 502 |
self.model, optimizer, train_dataloader
|
|
|
|
| 504 |
|
| 505 |
# Training loop
|
| 506 |
self.model.train()
|
| 507 |
+
global_step = resume_step
|
| 508 |
total_loss = 0
|
| 509 |
|
| 510 |
from tqdm import tqdm
|
| 511 |
progress_bar = tqdm(
|
| 512 |
total=max_steps,
|
| 513 |
+
initial=resume_step,
|
| 514 |
desc="SFT Training",
|
| 515 |
disable=not self.accelerator.is_local_main_process,
|
| 516 |
)
|
|
|
|
| 877 |
|
| 878 |
logger.info(f"Checkpoint saved to {save_path}")
|
| 879 |
|
| 880 |
+
def _find_latest_checkpoint(self, phase: str) -> Optional[Path]:
|
| 881 |
+
"""Find the latest checkpoint for a specific phase."""
|
| 882 |
+
checkpoint_dir = Path(self.config.output_dir)
|
| 883 |
+
if not checkpoint_dir.exists():
|
| 884 |
+
return None
|
| 885 |
+
|
| 886 |
+
# Find all checkpoints for this phase
|
| 887 |
+
checkpoints = list(checkpoint_dir.glob(f"checkpoint-{phase}-*"))
|
| 888 |
+
if not checkpoints:
|
| 889 |
+
return None
|
| 890 |
+
|
| 891 |
+
# Extract step numbers and find the latest
|
| 892 |
+
checkpoint_steps = []
|
| 893 |
+
for ckpt in checkpoints:
|
| 894 |
+
try:
|
| 895 |
+
step = int(ckpt.name.split("-")[-1])
|
| 896 |
+
checkpoint_steps.append((step, ckpt))
|
| 897 |
+
except ValueError:
|
| 898 |
+
continue
|
| 899 |
+
|
| 900 |
+
if not checkpoint_steps:
|
| 901 |
+
return None
|
| 902 |
+
|
| 903 |
+
# Return checkpoint with highest step number
|
| 904 |
+
latest_step, latest_path = max(checkpoint_steps, key=lambda x: x[0])
|
| 905 |
+
logger.info(f"Found checkpoint at step {latest_step}: {latest_path}")
|
| 906 |
+
return latest_path
|
| 907 |
+
|
| 908 |
+
def _load_checkpoint(self, checkpoint_path: Path, phase: str) -> int:
|
| 909 |
+
"""Load checkpoint and return the step number."""
|
| 910 |
+
if phase == "sft":
|
| 911 |
+
# Load model from checkpoint
|
| 912 |
+
from transformers import AutoModelForVision2Seq
|
| 913 |
+
logger.info(f"Loading SFT checkpoint from {checkpoint_path}")
|
| 914 |
+
|
| 915 |
+
# Load the checkpoint model
|
| 916 |
+
checkpoint_model = AutoModelForVision2Seq.from_pretrained(
|
| 917 |
+
checkpoint_path,
|
| 918 |
+
torch_dtype=torch.bfloat16,
|
| 919 |
+
device_map=self.accelerator.device,
|
| 920 |
+
)
|
| 921 |
+
|
| 922 |
+
# Copy weights to current model
|
| 923 |
+
self.model.load_state_dict(checkpoint_model.state_dict())
|
| 924 |
+
del checkpoint_model
|
| 925 |
+
|
| 926 |
+
elif phase == "prm":
|
| 927 |
+
logger.info(f"Loading PRM checkpoint from {checkpoint_path}")
|
| 928 |
+
self.prm.load_state_dict(
|
| 929 |
+
torch.load(checkpoint_path / "prm_model.pt", map_location=self.accelerator.device)
|
| 930 |
+
)
|
| 931 |
+
elif phase == "rl":
|
| 932 |
+
logger.info(f"Loading RL checkpoint from {checkpoint_path}")
|
| 933 |
+
self.rl_trainer.policy.load_state_dict(
|
| 934 |
+
torch.load(checkpoint_path / "rl_policy.pt", map_location=self.accelerator.device)
|
| 935 |
+
)
|
| 936 |
+
|
| 937 |
+
# Extract step number from checkpoint path
|
| 938 |
+
step = int(checkpoint_path.name.split("-")[-1])
|
| 939 |
+
logger.info(f"Resumed from checkpoint at step {step}")
|
| 940 |
+
return step
|
| 941 |
+
|
| 942 |
def _save_prm(self, epoch: int) -> None:
|
| 943 |
"""Save PRM checkpoint."""
|
| 944 |
save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"
|