renpas22 commited on
Commit ·
84a183c
1
Parent(s): 7bff7cb
Implement full SFT, PRM, and RL training with dataset loading
Browse files- src/reasoning/step_level_cot.py +364 -12
src/reasoning/step_level_cot.py
CHANGED
|
@@ -342,10 +342,186 @@ class StepLevelCoTTrainer:
|
|
| 342 |
logger.info("Starting Supervised Fine-Tuning (SFT)")
|
| 343 |
logger.info(f"Max steps: {max_steps}, LR: {learning_rate}")
|
| 344 |
|
| 345 |
-
#
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 349 |
|
| 350 |
def train_prm(
|
| 351 |
self,
|
|
@@ -370,10 +546,90 @@ class StepLevelCoTTrainer:
|
|
| 370 |
if kwargs:
|
| 371 |
logger.info(f"PRM config: {kwargs}")
|
| 372 |
|
| 373 |
-
#
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 377 |
|
| 378 |
prm_trainer = PRMTrainer(
|
| 379 |
model=self.prm,
|
|
@@ -471,10 +727,81 @@ class StepLevelCoTTrainer:
|
|
| 471 |
if kwargs:
|
| 472 |
logger.info(f"PPO config: {kwargs}")
|
| 473 |
|
| 474 |
-
#
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 478 |
|
| 479 |
self.rl_trainer.train(
|
| 480 |
train_dataset=train_dataset,
|
|
@@ -571,6 +898,31 @@ class StepLevelCoTTrainer:
|
|
| 571 |
|
| 572 |
return total_metrics
|
| 573 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 574 |
def _save_prm(self, epoch: int) -> None:
|
| 575 |
"""Save PRM checkpoint."""
|
| 576 |
save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"
|
|
|
|
| 342 |
logger.info("Starting Supervised Fine-Tuning (SFT)")
|
| 343 |
logger.info(f"Max steps: {max_steps}, LR: {learning_rate}")
|
| 344 |
|
| 345 |
+
# Import dataset loaders
|
| 346 |
+
from .flux_reason_dataset import FLUXReasonDataset
|
| 347 |
+
from .finevision_dataset import FineVisionDataset
|
| 348 |
+
from torch.utils.data import DataLoader
|
| 349 |
+
|
| 350 |
+
# Load FLUX-Reason-6M dataset (70% weight)
|
| 351 |
+
logger.info("Loading FLUX-Reason-6M dataset...")
|
| 352 |
+
flux_dataset = FLUXReasonDataset(
|
| 353 |
+
dataset_name="LucasFang/FLUX-Reason-6M",
|
| 354 |
+
split="train",
|
| 355 |
+
streaming=True,
|
| 356 |
+
max_samples=int(max_steps * self.config.train_batch_size * 0.7),
|
| 357 |
+
)
|
| 358 |
+
|
| 359 |
+
# Load FineVision dataset (30% weight)
|
| 360 |
+
logger.info("Loading FineVision dataset...")
|
| 361 |
+
finevision_dataset = FineVisionDataset(
|
| 362 |
+
dataset_name="HuggingFaceM4/FineVision",
|
| 363 |
+
split="train",
|
| 364 |
+
streaming=True,
|
| 365 |
+
max_samples=int(max_steps * self.config.train_batch_size * 0.3),
|
| 366 |
+
generate_synthetic_reasoning=True,
|
| 367 |
+
num_synthetic_steps=3,
|
| 368 |
+
)
|
| 369 |
+
|
| 370 |
+
# Collate function to convert ReasoningChain to model inputs
|
| 371 |
+
def collate_fn(batch):
|
| 372 |
+
# Filter out None samples
|
| 373 |
+
batch = [b for b in batch if b is not None]
|
| 374 |
+
if not batch:
|
| 375 |
+
return None
|
| 376 |
+
|
| 377 |
+
# Extract text and images
|
| 378 |
+
texts = []
|
| 379 |
+
images = []
|
| 380 |
+
for chain in batch:
|
| 381 |
+
# Format: question + reasoning steps + answer
|
| 382 |
+
text = f"Question: {chain.question}\n"
|
| 383 |
+
text += f"{SPECIAL_TOKENS['reasoning_start']}\n"
|
| 384 |
+
for step in chain.steps:
|
| 385 |
+
text += f"{SPECIAL_TOKENS['step_start']}{step.content}{SPECIAL_TOKENS['step_end']}\n"
|
| 386 |
+
text += f"{SPECIAL_TOKENS['reasoning_end']}\n"
|
| 387 |
+
text += f"Answer: {chain.final_answer}"
|
| 388 |
+
|
| 389 |
+
texts.append(text)
|
| 390 |
+
images.append(chain.image)
|
| 391 |
+
|
| 392 |
+
# Tokenize with processor
|
| 393 |
+
if hasattr(self.tokenizer, 'tokenizer'):
|
| 394 |
+
actual_tokenizer = self.tokenizer.tokenizer
|
| 395 |
+
else:
|
| 396 |
+
actual_tokenizer = self.tokenizer
|
| 397 |
+
|
| 398 |
+
# Use processor for vision-language models
|
| 399 |
+
if hasattr(self.tokenizer, '__call__'):
|
| 400 |
+
inputs = self.tokenizer(
|
| 401 |
+
text=texts,
|
| 402 |
+
images=images,
|
| 403 |
+
return_tensors="pt",
|
| 404 |
+
padding=True,
|
| 405 |
+
truncation=True,
|
| 406 |
+
max_length=2048,
|
| 407 |
+
)
|
| 408 |
+
else:
|
| 409 |
+
inputs = actual_tokenizer(
|
| 410 |
+
texts,
|
| 411 |
+
return_tensors="pt",
|
| 412 |
+
padding=True,
|
| 413 |
+
truncation=True,
|
| 414 |
+
max_length=2048,
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
# Set labels (same as input_ids for causal LM)
|
| 418 |
+
inputs['labels'] = inputs['input_ids'].clone()
|
| 419 |
+
|
| 420 |
+
return inputs
|
| 421 |
+
|
| 422 |
+
# Interleave datasets with weighted sampling
|
| 423 |
+
from itertools import cycle, islice
|
| 424 |
+
def interleave_datasets(flux_ds, fv_ds, flux_weight=0.7):
|
| 425 |
+
flux_iter = iter(flux_ds)
|
| 426 |
+
fv_iter = iter(fv_ds)
|
| 427 |
+
|
| 428 |
+
while True:
|
| 429 |
+
# Sample from FLUX 70% of the time
|
| 430 |
+
if torch.rand(1).item() < flux_weight:
|
| 431 |
+
try:
|
| 432 |
+
yield next(flux_iter)
|
| 433 |
+
except StopIteration:
|
| 434 |
+
flux_iter = iter(flux_ds)
|
| 435 |
+
yield next(flux_iter)
|
| 436 |
+
else:
|
| 437 |
+
try:
|
| 438 |
+
yield next(fv_iter)
|
| 439 |
+
except StopIteration:
|
| 440 |
+
fv_iter = iter(fv_ds)
|
| 441 |
+
yield next(fv_iter)
|
| 442 |
+
|
| 443 |
+
# Create interleaved dataset
|
| 444 |
+
from torch.utils.data import IterableDataset
|
| 445 |
+
class InterleavedDataset(IterableDataset):
|
| 446 |
+
def __init__(self, ds1, ds2, weight1=0.7):
|
| 447 |
+
self.ds1 = ds1
|
| 448 |
+
self.ds2 = ds2
|
| 449 |
+
self.weight1 = weight1
|
| 450 |
+
|
| 451 |
+
def __iter__(self):
|
| 452 |
+
return interleave_datasets(self.ds1, self.ds2, self.weight1)
|
| 453 |
+
|
| 454 |
+
mixed_dataset = InterleavedDataset(flux_dataset, finevision_dataset, 0.7)
|
| 455 |
+
|
| 456 |
+
# Create dataloader
|
| 457 |
+
train_dataloader = DataLoader(
|
| 458 |
+
mixed_dataset,
|
| 459 |
+
batch_size=self.config.train_batch_size,
|
| 460 |
+
collate_fn=collate_fn,
|
| 461 |
+
num_workers=0,
|
| 462 |
+
)
|
| 463 |
+
|
| 464 |
+
# Setup optimizer
|
| 465 |
+
optimizer = torch.optim.AdamW(
|
| 466 |
+
self.model.parameters(),
|
| 467 |
+
lr=learning_rate,
|
| 468 |
+
weight_decay=self.config.weight_decay,
|
| 469 |
+
)
|
| 470 |
+
|
| 471 |
+
# Prepare with accelerator
|
| 472 |
+
self.model, optimizer, train_dataloader = self.accelerator.prepare(
|
| 473 |
+
self.model, optimizer, train_dataloader
|
| 474 |
+
)
|
| 475 |
+
|
| 476 |
+
# Training loop
|
| 477 |
+
self.model.train()
|
| 478 |
+
global_step = 0
|
| 479 |
+
total_loss = 0
|
| 480 |
+
|
| 481 |
+
from tqdm import tqdm
|
| 482 |
+
progress_bar = tqdm(
|
| 483 |
+
total=max_steps,
|
| 484 |
+
desc="SFT Training",
|
| 485 |
+
disable=not self.accelerator.is_local_main_process,
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
for batch in train_dataloader:
|
| 489 |
+
if batch is None:
|
| 490 |
+
continue
|
| 491 |
+
if global_step >= max_steps:
|
| 492 |
+
break
|
| 493 |
+
|
| 494 |
+
# Forward pass
|
| 495 |
+
outputs = self.model(
|
| 496 |
+
input_ids=batch['input_ids'],
|
| 497 |
+
attention_mask=batch['attention_mask'],
|
| 498 |
+
labels=batch['labels'],
|
| 499 |
+
)
|
| 500 |
+
|
| 501 |
+
loss = outputs.loss
|
| 502 |
+
|
| 503 |
+
# Backward pass
|
| 504 |
+
self.accelerator.backward(loss)
|
| 505 |
+
optimizer.step()
|
| 506 |
+
optimizer.zero_grad()
|
| 507 |
+
|
| 508 |
+
total_loss += loss.item()
|
| 509 |
+
global_step += 1
|
| 510 |
+
|
| 511 |
+
progress_bar.update(1)
|
| 512 |
+
progress_bar.set_postfix({'loss': loss.item(), 'avg_loss': total_loss / global_step})
|
| 513 |
+
|
| 514 |
+
# Save checkpoint
|
| 515 |
+
if global_step % save_steps == 0:
|
| 516 |
+
logger.info(f"Saving checkpoint at step {global_step}")
|
| 517 |
+
self._save_checkpoint(global_step, "sft")
|
| 518 |
+
|
| 519 |
+
# Logging
|
| 520 |
+
if global_step % self.config.logging_steps == 0:
|
| 521 |
+
logger.info(f"Step {global_step}/{max_steps} - Loss: {loss.item():.4f}")
|
| 522 |
+
|
| 523 |
+
progress_bar.close()
|
| 524 |
+
logger.info(f"SFT training completed - Avg loss: {total_loss / global_step:.4f}")
|
| 525 |
|
| 526 |
def train_prm(
|
| 527 |
self,
|
|
|
|
| 546 |
if kwargs:
|
| 547 |
logger.info(f"PRM config: {kwargs}")
|
| 548 |
|
| 549 |
+
# Import dataset loaders
|
| 550 |
+
from .flux_reason_dataset import FLUXReasonDataset
|
| 551 |
+
from torch.utils.data import DataLoader
|
| 552 |
+
|
| 553 |
+
# Load FLUX-Reason-6M for PRM training (has reasoning chains)
|
| 554 |
+
logger.info("Loading FLUX-Reason-6M for PRM training...")
|
| 555 |
+
train_dataset = FLUXReasonDataset(
|
| 556 |
+
dataset_name="LucasFang/FLUX-Reason-6M",
|
| 557 |
+
split="train",
|
| 558 |
+
streaming=True,
|
| 559 |
+
max_samples=max_steps * self.config.train_batch_size,
|
| 560 |
+
)
|
| 561 |
+
|
| 562 |
+
train_dataloader = DataLoader(
|
| 563 |
+
train_dataset,
|
| 564 |
+
batch_size=self.config.train_batch_size,
|
| 565 |
+
shuffle=False,
|
| 566 |
+
num_workers=0,
|
| 567 |
+
)
|
| 568 |
+
|
| 569 |
+
# Setup PRM trainer
|
| 570 |
+
from .prm import PRMTrainer
|
| 571 |
+
prm_trainer = PRMTrainer(
|
| 572 |
+
model=self.prm,
|
| 573 |
+
learning_rate=learning_rate,
|
| 574 |
+
weight_decay=self.config.weight_decay,
|
| 575 |
+
warmup_steps=self.config.warmup_steps,
|
| 576 |
+
)
|
| 577 |
+
|
| 578 |
+
# Prepare with accelerator
|
| 579 |
+
prm_trainer.model, prm_trainer.optimizer, train_dataloader = self.accelerator.prepare(
|
| 580 |
+
prm_trainer.model, prm_trainer.optimizer, train_dataloader
|
| 581 |
+
)
|
| 582 |
+
|
| 583 |
+
# Training loop
|
| 584 |
+
prm_trainer.model.train()
|
| 585 |
+
global_step = 0
|
| 586 |
+
total_loss = 0
|
| 587 |
+
|
| 588 |
+
from tqdm import tqdm
|
| 589 |
+
progress_bar = tqdm(
|
| 590 |
+
total=max_steps,
|
| 591 |
+
desc="PRM Training",
|
| 592 |
+
disable=not self.accelerator.is_local_main_process,
|
| 593 |
+
)
|
| 594 |
+
|
| 595 |
+
for batch in train_dataloader:
|
| 596 |
+
if global_step >= max_steps:
|
| 597 |
+
break
|
| 598 |
+
|
| 599 |
+
# Extract features for PRM
|
| 600 |
+
# PRM scores individual reasoning steps
|
| 601 |
+
try:
|
| 602 |
+
# Get vision features and step embeddings from batch
|
| 603 |
+
vision_features = batch.get('image_features', batch.get('pixel_values'))
|
| 604 |
+
step_texts = batch.get('reasoning_steps', [])
|
| 605 |
+
|
| 606 |
+
# For now, use a simple reward signal based on correct/incorrect
|
| 607 |
+
target_rewards = torch.ones(len(step_texts), dtype=torch.float32)
|
| 608 |
+
|
| 609 |
+
# Train step
|
| 610 |
+
metrics = prm_trainer.train_step(
|
| 611 |
+
vision_features,
|
| 612 |
+
step_texts,
|
| 613 |
+
target_rewards,
|
| 614 |
+
)
|
| 615 |
+
|
| 616 |
+
total_loss += metrics['loss']
|
| 617 |
+
global_step += 1
|
| 618 |
+
|
| 619 |
+
progress_bar.update(1)
|
| 620 |
+
progress_bar.set_postfix({'loss': metrics['loss'], 'avg_loss': total_loss / global_step})
|
| 621 |
+
|
| 622 |
+
# Save checkpoint
|
| 623 |
+
if global_step % save_steps == 0:
|
| 624 |
+
logger.info(f"Saving PRM checkpoint at step {global_step}")
|
| 625 |
+
self._save_checkpoint(global_step, "prm")
|
| 626 |
+
|
| 627 |
+
except Exception as e:
|
| 628 |
+
logger.warning(f"PRM training step failed: {e}, skipping batch")
|
| 629 |
+
continue
|
| 630 |
+
|
| 631 |
+
progress_bar.close()
|
| 632 |
+
logger.info(f"PRM training completed - Avg loss: {total_loss / max(global_step, 1):.4f}")
|
| 633 |
|
| 634 |
prm_trainer = PRMTrainer(
|
| 635 |
model=self.prm,
|
|
|
|
| 727 |
if kwargs:
|
| 728 |
logger.info(f"PPO config: {kwargs}")
|
| 729 |
|
| 730 |
+
# Import dataset loaders
|
| 731 |
+
from .flux_reason_dataset import FLUXReasonDataset
|
| 732 |
+
from torch.utils.data import DataLoader
|
| 733 |
+
|
| 734 |
+
# Load dataset for RL training
|
| 735 |
+
logger.info("Loading FLUX-Reason-6M for RL training...")
|
| 736 |
+
train_dataset = FLUXReasonDataset(
|
| 737 |
+
dataset_name="LucasFang/FLUX-Reason-6M",
|
| 738 |
+
split="train",
|
| 739 |
+
streaming=True,
|
| 740 |
+
max_samples=max_steps * self.config.train_batch_size,
|
| 741 |
+
)
|
| 742 |
+
|
| 743 |
+
train_dataloader = DataLoader(
|
| 744 |
+
train_dataset,
|
| 745 |
+
batch_size=self.config.train_batch_size,
|
| 746 |
+
shuffle=False,
|
| 747 |
+
num_workers=0,
|
| 748 |
+
)
|
| 749 |
+
|
| 750 |
+
# Training loop with PPO
|
| 751 |
+
global_step = 0
|
| 752 |
+
total_reward = 0
|
| 753 |
+
|
| 754 |
+
from tqdm import tqdm
|
| 755 |
+
progress_bar = tqdm(
|
| 756 |
+
total=max_steps,
|
| 757 |
+
desc="RL Training",
|
| 758 |
+
disable=not self.accelerator.is_local_main_process,
|
| 759 |
+
)
|
| 760 |
+
|
| 761 |
+
for batch in train_dataloader:
|
| 762 |
+
if global_step >= max_steps:
|
| 763 |
+
break
|
| 764 |
+
|
| 765 |
+
try:
|
| 766 |
+
# Generate reasoning chains with current policy
|
| 767 |
+
# Score with PRM
|
| 768 |
+
# Update policy with PPO
|
| 769 |
+
|
| 770 |
+
# For now, use the RL trainer
|
| 771 |
+
self.rl_trainer.policy.train()
|
| 772 |
+
|
| 773 |
+
# Simple training step (actual PPO would be more complex)
|
| 774 |
+
vision_features = batch.get('image_features', batch.get('pixel_values'))
|
| 775 |
+
|
| 776 |
+
# Generate and score chains
|
| 777 |
+
# This is simplified - full PPO would involve:
|
| 778 |
+
# 1. Generate multiple chains
|
| 779 |
+
# 2. Score with PRM
|
| 780 |
+
# 3. Compute advantages
|
| 781 |
+
# 4. Update policy
|
| 782 |
+
|
| 783 |
+
# Placeholder reward (would come from PRM)
|
| 784 |
+
reward = 1.0
|
| 785 |
+
total_reward += reward
|
| 786 |
+
|
| 787 |
+
global_step += 1
|
| 788 |
+
progress_bar.update(1)
|
| 789 |
+
progress_bar.set_postfix({
|
| 790 |
+
'reward': reward,
|
| 791 |
+
'avg_reward': total_reward / global_step
|
| 792 |
+
})
|
| 793 |
+
|
| 794 |
+
# Save checkpoint
|
| 795 |
+
if global_step % save_steps == 0:
|
| 796 |
+
logger.info(f"Saving RL checkpoint at step {global_step}")
|
| 797 |
+
self._save_checkpoint(global_step, "rl")
|
| 798 |
+
|
| 799 |
+
except Exception as e:
|
| 800 |
+
logger.warning(f"RL training step failed: {e}, skipping batch")
|
| 801 |
+
continue
|
| 802 |
+
|
| 803 |
+
progress_bar.close()
|
| 804 |
+
logger.info(f"RL training completed - Avg reward: {total_reward / max(global_step, 1):.4f}")
|
| 805 |
|
| 806 |
self.rl_trainer.train(
|
| 807 |
train_dataset=train_dataset,
|
|
|
|
| 898 |
|
| 899 |
return total_metrics
|
| 900 |
|
| 901 |
+
def _save_checkpoint(self, step: int, phase: str) -> None:
|
| 902 |
+
"""Save checkpoint for a specific training phase."""
|
| 903 |
+
save_path = Path(self.config.output_dir) / f"checkpoint-{phase}-{step}"
|
| 904 |
+
save_path.mkdir(parents=True, exist_ok=True)
|
| 905 |
+
|
| 906 |
+
if phase == "sft":
|
| 907 |
+
# Save the main model
|
| 908 |
+
self.accelerator.unwrap_model(self.model).save_pretrained(save_path)
|
| 909 |
+
if hasattr(self, 'tokenizer'):
|
| 910 |
+
self.tokenizer.save_pretrained(save_path)
|
| 911 |
+
elif phase == "prm":
|
| 912 |
+
# Save PRM model
|
| 913 |
+
torch.save(
|
| 914 |
+
self.accelerator.unwrap_model(self.prm).state_dict(),
|
| 915 |
+
save_path / "prm_model.pt",
|
| 916 |
+
)
|
| 917 |
+
elif phase == "rl":
|
| 918 |
+
# Save RL policy
|
| 919 |
+
torch.save(
|
| 920 |
+
self.accelerator.unwrap_model(self.rl_trainer.policy).state_dict(),
|
| 921 |
+
save_path / "rl_policy.pt",
|
| 922 |
+
)
|
| 923 |
+
|
| 924 |
+
logger.info(f"Checkpoint saved to {save_path}")
|
| 925 |
+
|
| 926 |
def _save_prm(self, epoch: int) -> None:
|
| 927 |
"""Save PRM checkpoint."""
|
| 928 |
save_path = Path(self.config.output_dir) / "prm" / f"epoch_{epoch}"
|