ctopd / kdflow /cli /train_kd_off_policy.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
5.88 kB
import os
import ray
from kdflow.ray.train.teacher_group import TeacherActorGroup
from kdflow.ray.train.student_group import StudentActorGroup
from kdflow.ray.placement_group import create_placement_group
from kdflow.trainer import OffPolicyKDTrainer
from kdflow.datasets import SFTDataset
from kdflow.datasets.utils import blending_datasets
from kdflow.models.utils import check_tokenizer_identical
from kdflow.backend import get_strategy
from kdflow.arguments import init_args
from kdflow.arguments.training_args import resolve_max_steps
from kdflow.utils.distributed_sampler import DistributedSampler
from kdflow.utils.utils import get_tokenizer
def train(args):
# Initialize Ray if not already initialized
if not ray.is_initialized():
ray.init(
runtime_env={
"env_vars": {
"TOKENIZERS_PARALLELISM": "true",
"NCCL_DEBUG": "WARN"
},
"working_dir": os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
}
)
strategy = get_strategy(args)
strategy.print(args)
# Create placement group for resource allocation
num_gpus = args.train.num_nodes * args.train.num_gpus_per_node
pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(num_gpus)
strategy.print(f"Placement group ready with {num_gpus} GPUs")
teacher_model = TeacherActorGroup(
strategy,
num_gpus,
num_gpus_per_node=args.train.num_gpus_per_node,
num_gpus_per_actor=0.01,
pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
)
student_model = StudentActorGroup(
args,
args.train.num_nodes,
args.train.num_gpus_per_node,
pg=pg,
num_gpus_per_actor=0.01,
)
# Initialize tokenizers
student_tokenizer = get_tokenizer(
args.model.student_name_or_path,
use_fast=not args.model.disable_fast_tokenizer
)
teacher_tokenizer = get_tokenizer(
args.model.teacher_name_or_path,
use_fast=not args.model.disable_fast_tokenizer,
)
tokenizer_info = check_tokenizer_identical(student_tokenizer, teacher_tokenizer)
strategy.print(f"Tokenizers {tokenizer_info}")
if not tokenizer_info.vocab_identical and args.kd.kd_algorithm not in {
"simple_ctkd",
"byte_align_ctkd",
"topk_byte_align_ctkd",
}:
raise ValueError(
"Cross-tokenizer distillation supports simple_ctkd, "
"byte_align_ctkd, or topk_byte_align_ctkd."
)
# Load and prepare training dataset
train_data = blending_datasets(
args.data.train_dataset_path,
args.data.train_dataset_probs,
strategy,
args.train.seed,
max_count=args.data.max_samples,
dataset_split=args.data.train_split,
)
train_data = train_data.select(range(min(args.data.max_samples, len(train_data))))
train_dataset = SFTDataset(
train_data,
strategy,
tokenizer_info=tokenizer_info,
max_data_num=args.data.max_samples,
input_template=args.data.input_template,
num_processors=args.data.preprocess_num_workers,
)
sampler = DistributedSampler(train_dataset, num_replicas=1, rank=0, shuffle=True, seed=args.train.seed, drop_last=True)
train_dataloader = strategy.setup_dataloader(
train_dataset,
args.train.micro_train_batch_size,
pin_memory=False,
shuffle=False,
collate_fn=train_dataset.collate_fn,
sampler=sampler,
)
# Load and prepare evaluation dataset (optional)
eval_dataloader = None
if getattr(args.data, "eval_dataset_path", None):
eval_data = blending_datasets(
args.data.eval_dataset_path,
None,
strategy,
dataset_split=args.data.eval_split,
)
eval_data = eval_data.select(range(min(args.data.max_samples, len(eval_data))))
eval_dataset = SFTDataset(
eval_data,
strategy,
tokenizer_info=tokenizer_info,
input_template=args.data.input_template,
num_processors=args.data.preprocess_num_workers,
)
eval_dataloader = strategy.setup_dataloader(
eval_dataset,
batch_size=1,
pin_memory=True,
shuffle=False,
drop_last=False,
collate_fn=eval_dataset.collate_fn,
)
# Calculate max training steps
num_update_steps_per_epoch = len(train_dataset) // args.train.train_batch_size
max_steps = resolve_max_steps(
args.train.max_steps,
args.train.num_epochs,
num_update_steps_per_epoch,
)
strategy.log(f"Max training steps: {max_steps}")
# Initialize student model on all workers
ray.get(student_model.async_init_model_from_pretrained(
strategy, max_steps,
teacher_tokenizer=teacher_tokenizer,
tokenizer_info=tokenizer_info,
))
strategy.log("Models initialized on all student actors")
# Initialize OffPolicyKDRayTrainer (Single controller)
trainer = OffPolicyKDTrainer(
strategy=strategy,
student_model=student_model,
teacher_model=teacher_model,
train_dataloader=train_dataloader,
eval_dataloader=eval_dataloader,
max_steps=max_steps,
num_update_steps_per_epoch=num_update_steps_per_epoch,
student_tokenizer=student_tokenizer,
teacher_tokenizer=teacher_tokenizer,
)
try:
trainer.fit()
ray.get(student_model.async_save_model())
strategy.log("Training completed and model saved.")
finally:
teacher_model.shutdown()
if __name__ == "__main__":
args = init_args("off_policy_kd")
train(args)