Download kdflow/trainer/off_policy_kd_trainer.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 14 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/trainer/off_policy_kd_trainer.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/trainer/off_policy_kd_trainer.py
-
curl -L -o off_policy_kd_trainer.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/trainer/off_policy_kd_trainer.py
14 kB
| import os | |
| import time | |
| import json | |
| import math | |
| from datetime import timedelta | |
| from typing import Dict, List, Optional, Callable, Any | |
| from collections import defaultdict | |
| import ray | |
| import torch | |
| import torch.distributed as dist | |
| from tqdm import tqdm | |
| from kdflow.utils.logging_utils import define_wandb_metrics, init_logger, log_eval_metrics | |
| from kdflow.utils.dynamic_bsz import rearrange_global_batch | |
| logger = init_logger(__name__) | |
| class OffPolicyKDTrainer: | |
| """ | |
| Ray-based trainer for off-policy knowledge distillation. | |
| """ | |
| def __init__( | |
| self, | |
| strategy, | |
| student_model, | |
| teacher_model, | |
| train_dataloader, | |
| eval_dataloader=None, | |
| max_steps: int = None, | |
| num_update_steps_per_epoch: int = None, | |
| student_tokenizer=None, | |
| teacher_tokenizer=None, | |
| ) -> None: | |
| """ | |
| Initialize the trainer. | |
| Args: | |
| strategy: Training strategy containing configuration | |
| student_model: StudentActorGroup | |
| teacher_model: TeacherActorGroup | |
| train_dataloader: Training data loader | |
| eval_dataloader: Evaluation data loader (optional) | |
| max_steps: Maximum training steps | |
| num_update_steps_per_epoch: Number of update steps per epoch | |
| """ | |
| self.strategy = strategy | |
| self.args = strategy.args | |
| self.student = student_model | |
| self.teacher = teacher_model | |
| self.train_dataloader = train_dataloader | |
| self.eval_dataloader = eval_dataloader | |
| self.num_update_steps_per_epoch = num_update_steps_per_epoch | |
| self.max_steps = ( | |
| max_steps | |
| if max_steps is not None and max_steps > 0 | |
| else self.args.train.num_epochs * self.num_update_steps_per_epoch | |
| ) | |
| self.student_tokenizer = student_tokenizer | |
| self.teacher_tokenizer = teacher_tokenizer | |
| self.epochs = math.ceil( | |
| self.max_steps / self.num_update_steps_per_epoch | |
| ) | |
| self.world_size = self.args.train.num_nodes * self.args.train.num_gpus_per_node | |
| self.dp_size = self.world_size // self.args.model.ring_attn_size | |
| self.log_state = defaultdict(list) | |
| self._init_loggers() | |
| if self.eval_dataloader and self.args.train.eval_steps < float("inf"): | |
| assert ( | |
| self.args.train.eval_steps >= self.args.kd.teacher_forward_n_batches | |
| and self.args.train.eval_steps % self.args.kd.teacher_forward_n_batches == 0 | |
| ), ( | |
| "`eval_steps` must be a multiple of `teacher_forward_n_batches` " | |
| f"and no smaller than it, but got eval_steps={self.args.train.eval_steps}, " | |
| f"teacher_forward_n_batches={self.args.kd.teacher_forward_n_batches}." | |
| ) | |
| def _init_loggers(self) -> None: | |
| """Initialize wandb loggers.""" | |
| self._wandb = None | |
| if self.args.log.use_wandb: | |
| import wandb | |
| self._wandb = wandb | |
| if self.args.log.wandb_mode != "offline" and not wandb.api.api_key: | |
| wandb.login() | |
| wandb.init( | |
| entity=self.args.log.wandb_org, | |
| project=self.args.log.wandb_project, | |
| group=self.args.log.wandb_group, | |
| name=self.args.log.wandb_run_name, | |
| config=vars(self.args), | |
| reinit=True, | |
| mode=self.args.log.wandb_mode, | |
| dir=self.args.log.wandb_dir, | |
| ) | |
| define_wandb_metrics(wandb) | |
| def _print_training_config(self) -> None: | |
| """Log training configuration before training starts.""" | |
| total_steps = self.max_steps | |
| num_data = len(getattr(self.train_dataloader, "dataset", self.train_dataloader)) | |
| grad_accum = self.args.train.train_batch_size * self.args.model.ring_attn_size \ | |
| // (self.args.train.micro_train_batch_size * self.args.train.num_nodes * self.args.train.num_gpus_per_node) | |
| def log_config(name, value): | |
| logger.info(f" {name:<32} {value}") | |
| logger.info("******* Start Training *******") | |
| log_config("Num GPUs:", self.world_size) | |
| log_config("Num Data:", num_data) | |
| log_config("Num Epochs:", self.epochs) | |
| log_config("Train Batch Size:", self.args.train.train_batch_size) | |
| log_config("Steps Per Epoch:", self.num_update_steps_per_epoch) | |
| log_config("Total Training Steps:", total_steps) | |
| if self.args.train.use_dynamic_bsz: | |
| log_config("Enable Dynamic Batch Size:", self.args.train.use_dynamic_bsz) | |
| log_config("Max Token Len Per GPU:", self.args.train.max_token_len_per_gpu) | |
| log_config("Gradient Accumulation:", "dynamic") | |
| else: | |
| log_config("Per-device Batch Size:", self.args.train.micro_train_batch_size) | |
| log_config("Gradient Accumulation:", grad_accum) | |
| log_config("Learning Rate:", self.args.train.learning_rate) | |
| log_config("KD Algorithm:", self.args.kd.kd_algorithm) | |
| log_config("KD Loss Function:", self.args.kd.kd_loss_fn) | |
| if self.args.kd.kd_algorithm == "topk_byte_align_ctkd": | |
| log_config("CTKD Student Top-K:", self.args.kd.ctkd_topk) | |
| def fit(self, global_step=0, start_epoch=0): | |
| self.global_step = global_step | |
| self._print_training_config() | |
| self.start_time = time.time() | |
| num_micro_batches = self.args.train.train_batch_size // self.args.train.micro_train_batch_size | |
| self.teacher_forward_n = min(self.args.kd.teacher_forward_n_batches, len(self.train_dataloader)) | |
| teacher_forward_n = self.teacher_forward_n | |
| if self.eval_dataloader is not None and self.args.train.eval_steps < float("inf") and self.global_step == 0: | |
| self.strategy.log(f"Start evaluating at global step {self.global_step}") | |
| self.evaluate() | |
| for epoch in range(start_epoch, self.epochs): | |
| self.current_epoch = epoch | |
| self.train_dataloader.sampler.set_epoch(epoch) | |
| data_iter = iter(self.train_dataloader) | |
| is_epoch_finished = False | |
| while True: | |
| if self.global_step >= self.max_steps: | |
| break | |
| step_group_start = time.time() | |
| all_global_batches = [] | |
| steps_to_collect = min( | |
| teacher_forward_n, | |
| self.max_steps - self.global_step, | |
| ) | |
| for _ in range(steps_to_collect): | |
| global_batch = [] | |
| try: | |
| for _ in range(num_micro_batches): | |
| micro_batch = next(data_iter) | |
| global_batch.append(micro_batch) | |
| except StopIteration: | |
| is_epoch_finished = True | |
| break | |
| if not is_epoch_finished: | |
| if self.args.train.use_dynamic_bsz: | |
| global_batch = rearrange_global_batch( | |
| global_batch, | |
| max_token_len=self.args.train.max_token_len_per_gpu, | |
| dp_size=self.dp_size, | |
| ) | |
| global_batch_token_num = sum(mb["stu_loss_mask"].sum() for mb in global_batch) | |
| avg_micro_batch_token_num = global_batch_token_num / len(global_batch) | |
| for mb in global_batch: | |
| mb["avg_micro_batch_token_num"] = avg_micro_batch_token_num | |
| all_global_batches.append(global_batch) | |
| if not all_global_batches: | |
| break | |
| # ===== Teacher Phase (batch N global batches) ===== | |
| teacher_start = time.time() | |
| if self.args.train.enable_sleep: | |
| self.teacher.wakeup() | |
| merged_batch = [mb for gb in all_global_batches for mb in gb] | |
| merged_batch = self.teacher.forward(merged_batch) | |
| idx = 0 | |
| for i, gb in enumerate(all_global_batches): | |
| all_global_batches[i] = merged_batch[idx:idx + len(gb)] | |
| idx += len(gb) | |
| if self.args.train.enable_sleep: | |
| self.teacher.sleep() | |
| teacher_step_fwd_time = (time.time() - teacher_start) / len(all_global_batches) | |
| # ===== Student Phase (train N steps) ===== | |
| if self.args.train.enable_sleep: | |
| self.student.wakeup() | |
| shared_step_time = (time.time() - step_group_start) / len(all_global_batches) | |
| for global_batch in all_global_batches: | |
| student_start = time.time() | |
| self.global_step += 1 | |
| status_list = ray.get(self.student.async_run_distill(global_batch)) | |
| student_step_train_time = time.time() - student_start | |
| for k in status_list[0].keys(): | |
| self.log_state[k].append(sum(s[k] for s in status_list) / len(status_list)) | |
| self.log_state["timing/teacher_forward_time"].append(teacher_step_fwd_time) | |
| self.log_state["timing/student_train"].append(student_step_train_time) | |
| self.log_state["timing/step_time"].append(shared_step_time + student_step_train_time) | |
| self.logging() | |
| if self.global_step % self.args.train.save_steps == 0: | |
| self.strategy.log(f"Saving model at global step {self.global_step}") | |
| save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}_global_step_{self.global_step}") | |
| ray.get(self.student.async_save_model(save_path)) | |
| if self.args.train.enable_sleep: | |
| self.student.sleep() | |
| if ( | |
| self.eval_dataloader is not None | |
| and self.global_step % self.args.train.eval_steps == 0 | |
| ): | |
| self.strategy.log(f"Start evaluating at global step {self.global_step}") | |
| self.evaluate() | |
| if self.global_step >= self.max_steps: | |
| self.strategy.log( | |
| f"Reached max_steps={self.max_steps}; stopping training." | |
| ) | |
| break | |
| self.strategy.log(f"Saving model after epoch {epoch + 1}") | |
| save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}") | |
| ray.get(self.student.async_save_model(save_path)) | |
| total_time = time.time() - self.start_time | |
| self.strategy.log(f"Training done, totally cost {str(timedelta(seconds=total_time)).split('.')[0]}") | |
| if self._wandb is not None: | |
| self._wandb.finish() | |
| def evaluate(self): | |
| """Evaluate KD loss and distillation metrics without updating the student.""" | |
| eval_batches = list(self.eval_dataloader) | |
| if not eval_batches: | |
| return {} | |
| if self.args.train.use_dynamic_bsz: | |
| eval_batches = rearrange_global_batch( | |
| eval_batches, | |
| max_token_len=self.args.train.max_token_len_per_gpu, | |
| dp_size=self.dp_size, | |
| ) | |
| if self.args.train.enable_sleep: | |
| self.teacher.wakeup() | |
| eval_batches = self.teacher.forward(eval_batches) | |
| if self.args.train.enable_sleep: | |
| self.teacher.sleep() | |
| if self.args.train.enable_sleep: | |
| self.student.wakeup() | |
| metrics = ray.get(self.student.async_run_eval(eval_batches))[0] | |
| if self.args.train.enable_sleep: | |
| self.student.sleep() | |
| return log_eval_metrics(self.strategy, self._wandb, metrics, self.global_step) | |
| def logging(self): | |
| if self.global_step % self.args.log.logging_steps == 0: | |
| progress = self.global_step / self.max_steps | |
| eta = int(time.time() - self.start_time) * (1 - progress) / progress | |
| progress_str = "epoch [{current_epoch}/{total_epoch}], " \ | |
| "step [{current_step}/{total_step}], " \ | |
| "train_progress [{progress:.2f}%], " \ | |
| "Elapsed: {elapsed}, " \ | |
| "ETA: {eta}, ".format( | |
| current_epoch=self.current_epoch + 1, | |
| total_epoch=self.epochs, | |
| current_step=self.global_step, | |
| total_step=self.max_steps, | |
| progress=progress * 100, | |
| elapsed=str(timedelta(seconds=(time.time() - self.start_time))).split(".")[0], | |
| eta=str(timedelta(seconds=eta)).split(".")[0] | |
| ) | |
| for k in self.log_state: | |
| if isinstance(self.log_state[k], list) and len(self.log_state[k]) > 0: | |
| self.log_state[k] = sum(self.log_state[k]) / len(self.log_state[k]) | |
| log_info = [] | |
| for k in self.log_state: | |
| if k == "train/lr": | |
| log_info.append(f"{k}: {self.log_state[k]:.6e}") | |
| else: | |
| log_info.append(f"{k}: {self.log_state[k]:.6f}") | |
| log_str = ", ".join(log_info) | |
| log_str = progress_str + log_str | |
| self.strategy.log(log_str) | |
| if self._wandb is not None: | |
| logs = {"train/global_step": self.global_step} | |
| for k in self.log_state: | |
| logs[k] = self.log_state[k] | |
| self._wandb.log(logs) | |
| for k in self.log_state: | |
| self.log_state[k] = [] | |