ctopd / kdflow /trainer /off_policy_kd_trainer.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
14 kB
import os
import time
import json
import math
from datetime import timedelta
from typing import Dict, List, Optional, Callable, Any
from collections import defaultdict
import ray
import torch
import torch.distributed as dist
from tqdm import tqdm
from kdflow.utils.logging_utils import define_wandb_metrics, init_logger, log_eval_metrics
from kdflow.utils.dynamic_bsz import rearrange_global_batch
logger = init_logger(__name__)
class OffPolicyKDTrainer:
"""
Ray-based trainer for off-policy knowledge distillation.
"""
def __init__(
self,
strategy,
student_model,
teacher_model,
train_dataloader,
eval_dataloader=None,
max_steps: int = None,
num_update_steps_per_epoch: int = None,
student_tokenizer=None,
teacher_tokenizer=None,
) -> None:
"""
Initialize the trainer.
Args:
strategy: Training strategy containing configuration
student_model: StudentActorGroup
teacher_model: TeacherActorGroup
train_dataloader: Training data loader
eval_dataloader: Evaluation data loader (optional)
max_steps: Maximum training steps
num_update_steps_per_epoch: Number of update steps per epoch
"""
self.strategy = strategy
self.args = strategy.args
self.student = student_model
self.teacher = teacher_model
self.train_dataloader = train_dataloader
self.eval_dataloader = eval_dataloader
self.num_update_steps_per_epoch = num_update_steps_per_epoch
self.max_steps = (
max_steps
if max_steps is not None and max_steps > 0
else self.args.train.num_epochs * self.num_update_steps_per_epoch
)
self.student_tokenizer = student_tokenizer
self.teacher_tokenizer = teacher_tokenizer
self.epochs = math.ceil(
self.max_steps / self.num_update_steps_per_epoch
)
self.world_size = self.args.train.num_nodes * self.args.train.num_gpus_per_node
self.dp_size = self.world_size // self.args.model.ring_attn_size
self.log_state = defaultdict(list)
self._init_loggers()
if self.eval_dataloader and self.args.train.eval_steps < float("inf"):
assert (
self.args.train.eval_steps >= self.args.kd.teacher_forward_n_batches
and self.args.train.eval_steps % self.args.kd.teacher_forward_n_batches == 0
), (
"`eval_steps` must be a multiple of `teacher_forward_n_batches` "
f"and no smaller than it, but got eval_steps={self.args.train.eval_steps}, "
f"teacher_forward_n_batches={self.args.kd.teacher_forward_n_batches}."
)
def _init_loggers(self) -> None:
"""Initialize wandb loggers."""
self._wandb = None
if self.args.log.use_wandb:
import wandb
self._wandb = wandb
if self.args.log.wandb_mode != "offline" and not wandb.api.api_key:
wandb.login()
wandb.init(
entity=self.args.log.wandb_org,
project=self.args.log.wandb_project,
group=self.args.log.wandb_group,
name=self.args.log.wandb_run_name,
config=vars(self.args),
reinit=True,
mode=self.args.log.wandb_mode,
dir=self.args.log.wandb_dir,
)
define_wandb_metrics(wandb)
def _print_training_config(self) -> None:
"""Log training configuration before training starts."""
total_steps = self.max_steps
num_data = len(getattr(self.train_dataloader, "dataset", self.train_dataloader))
grad_accum = self.args.train.train_batch_size * self.args.model.ring_attn_size \
// (self.args.train.micro_train_batch_size * self.args.train.num_nodes * self.args.train.num_gpus_per_node)
def log_config(name, value):
logger.info(f" {name:<32} {value}")
logger.info("******* Start Training *******")
log_config("Num GPUs:", self.world_size)
log_config("Num Data:", num_data)
log_config("Num Epochs:", self.epochs)
log_config("Train Batch Size:", self.args.train.train_batch_size)
log_config("Steps Per Epoch:", self.num_update_steps_per_epoch)
log_config("Total Training Steps:", total_steps)
if self.args.train.use_dynamic_bsz:
log_config("Enable Dynamic Batch Size:", self.args.train.use_dynamic_bsz)
log_config("Max Token Len Per GPU:", self.args.train.max_token_len_per_gpu)
log_config("Gradient Accumulation:", "dynamic")
else:
log_config("Per-device Batch Size:", self.args.train.micro_train_batch_size)
log_config("Gradient Accumulation:", grad_accum)
log_config("Learning Rate:", self.args.train.learning_rate)
log_config("KD Algorithm:", self.args.kd.kd_algorithm)
log_config("KD Loss Function:", self.args.kd.kd_loss_fn)
if self.args.kd.kd_algorithm == "topk_byte_align_ctkd":
log_config("CTKD Student Top-K:", self.args.kd.ctkd_topk)
def fit(self, global_step=0, start_epoch=0):
self.global_step = global_step
self._print_training_config()
self.start_time = time.time()
num_micro_batches = self.args.train.train_batch_size // self.args.train.micro_train_batch_size
self.teacher_forward_n = min(self.args.kd.teacher_forward_n_batches, len(self.train_dataloader))
teacher_forward_n = self.teacher_forward_n
if self.eval_dataloader is not None and self.args.train.eval_steps < float("inf") and self.global_step == 0:
self.strategy.log(f"Start evaluating at global step {self.global_step}")
self.evaluate()
for epoch in range(start_epoch, self.epochs):
self.current_epoch = epoch
self.train_dataloader.sampler.set_epoch(epoch)
data_iter = iter(self.train_dataloader)
is_epoch_finished = False
while True:
if self.global_step >= self.max_steps:
break
step_group_start = time.time()
all_global_batches = []
steps_to_collect = min(
teacher_forward_n,
self.max_steps - self.global_step,
)
for _ in range(steps_to_collect):
global_batch = []
try:
for _ in range(num_micro_batches):
micro_batch = next(data_iter)
global_batch.append(micro_batch)
except StopIteration:
is_epoch_finished = True
break
if not is_epoch_finished:
if self.args.train.use_dynamic_bsz:
global_batch = rearrange_global_batch(
global_batch,
max_token_len=self.args.train.max_token_len_per_gpu,
dp_size=self.dp_size,
)
global_batch_token_num = sum(mb["stu_loss_mask"].sum() for mb in global_batch)
avg_micro_batch_token_num = global_batch_token_num / len(global_batch)
for mb in global_batch:
mb["avg_micro_batch_token_num"] = avg_micro_batch_token_num
all_global_batches.append(global_batch)
if not all_global_batches:
break
# ===== Teacher Phase (batch N global batches) =====
teacher_start = time.time()
if self.args.train.enable_sleep:
self.teacher.wakeup()
merged_batch = [mb for gb in all_global_batches for mb in gb]
merged_batch = self.teacher.forward(merged_batch)
idx = 0
for i, gb in enumerate(all_global_batches):
all_global_batches[i] = merged_batch[idx:idx + len(gb)]
idx += len(gb)
if self.args.train.enable_sleep:
self.teacher.sleep()
teacher_step_fwd_time = (time.time() - teacher_start) / len(all_global_batches)
# ===== Student Phase (train N steps) =====
if self.args.train.enable_sleep:
self.student.wakeup()
shared_step_time = (time.time() - step_group_start) / len(all_global_batches)
for global_batch in all_global_batches:
student_start = time.time()
self.global_step += 1
status_list = ray.get(self.student.async_run_distill(global_batch))
student_step_train_time = time.time() - student_start
for k in status_list[0].keys():
self.log_state[k].append(sum(s[k] for s in status_list) / len(status_list))
self.log_state["timing/teacher_forward_time"].append(teacher_step_fwd_time)
self.log_state["timing/student_train"].append(student_step_train_time)
self.log_state["timing/step_time"].append(shared_step_time + student_step_train_time)
self.logging()
if self.global_step % self.args.train.save_steps == 0:
self.strategy.log(f"Saving model at global step {self.global_step}")
save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}_global_step_{self.global_step}")
ray.get(self.student.async_save_model(save_path))
if self.args.train.enable_sleep:
self.student.sleep()
if (
self.eval_dataloader is not None
and self.global_step % self.args.train.eval_steps == 0
):
self.strategy.log(f"Start evaluating at global step {self.global_step}")
self.evaluate()
if self.global_step >= self.max_steps:
self.strategy.log(
f"Reached max_steps={self.max_steps}; stopping training."
)
break
self.strategy.log(f"Saving model after epoch {epoch + 1}")
save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}")
ray.get(self.student.async_save_model(save_path))
total_time = time.time() - self.start_time
self.strategy.log(f"Training done, totally cost {str(timedelta(seconds=total_time)).split('.')[0]}")
if self._wandb is not None:
self._wandb.finish()
def evaluate(self):
"""Evaluate KD loss and distillation metrics without updating the student."""
eval_batches = list(self.eval_dataloader)
if not eval_batches:
return {}
if self.args.train.use_dynamic_bsz:
eval_batches = rearrange_global_batch(
eval_batches,
max_token_len=self.args.train.max_token_len_per_gpu,
dp_size=self.dp_size,
)
if self.args.train.enable_sleep:
self.teacher.wakeup()
eval_batches = self.teacher.forward(eval_batches)
if self.args.train.enable_sleep:
self.teacher.sleep()
if self.args.train.enable_sleep:
self.student.wakeup()
metrics = ray.get(self.student.async_run_eval(eval_batches))[0]
if self.args.train.enable_sleep:
self.student.sleep()
return log_eval_metrics(self.strategy, self._wandb, metrics, self.global_step)
def logging(self):
if self.global_step % self.args.log.logging_steps == 0:
progress = self.global_step / self.max_steps
eta = int(time.time() - self.start_time) * (1 - progress) / progress
progress_str = "epoch [{current_epoch}/{total_epoch}], " \
"step [{current_step}/{total_step}], " \
"train_progress [{progress:.2f}%], " \
"Elapsed: {elapsed}, " \
"ETA: {eta}, ".format(
current_epoch=self.current_epoch + 1,
total_epoch=self.epochs,
current_step=self.global_step,
total_step=self.max_steps,
progress=progress * 100,
elapsed=str(timedelta(seconds=(time.time() - self.start_time))).split(".")[0],
eta=str(timedelta(seconds=eta)).split(".")[0]
)
for k in self.log_state:
if isinstance(self.log_state[k], list) and len(self.log_state[k]) > 0:
self.log_state[k] = sum(self.log_state[k]) / len(self.log_state[k])
log_info = []
for k in self.log_state:
if k == "train/lr":
log_info.append(f"{k}: {self.log_state[k]:.6e}")
else:
log_info.append(f"{k}: {self.log_state[k]:.6f}")
log_str = ", ".join(log_info)
log_str = progress_str + log_str
self.strategy.log(log_str)
if self._wandb is not None:
logs = {"train/global_step": self.global_step}
for k in self.log_state:
logs[k] = self.log_state[k]
self._wandb.log(logs)
for k in self.log_state:
self.log_state[k] = []