from slyme.utils.registry import Registry from slyme.context import ARG, Arg, R from slyme.node import Node from slyme.builder import builder from thinkstream.trainer.expressions import get_attribute from thinkstream.data.stream_data_processor import ( DEFAULT_INFERENCE_MAX_PIXELS, DEFAULT_INFERENCE_MIN_PIXELS, DEFAULT_MAX_CHUNKS, ) from deepslyme.node.common import ( set_seed, free_memory, empty_cache_by_step as empty_cache, prepare_distributed_dataloader, prepare_inputs, clean_step_inputs, setup_dtype, create_optimizer, create_scheduler, init_progress, update_progress, destroy_progress, dataloader_loop, dataloader_loop_with_micro_steps, ) from deepslyme.node.integration.deepspeed import ( deepspeed_init_distributed, deepspeed_config_init, deepspeed_backward, deepspeed_step, deepspeed_initialize, deepspeed_set_grad_acc_boundary, deepspeed_with_grad_acc_boundary, ) from deepslyme.node.integration.liger_kernel import apply_liger_kernel from deepslyme.node.sft import ( init_training_state, epoch_loop, dataloader_set_epoch, compute_loss, ) from deepslyme.node.metric import collect_metrics, reduce_and_log_metrics from deepslyme.node.rl.grpo import calc_grpo_advantages from thinkstream.trainer.sft import ( report_distributed_runtime, build_optimizer_kwargs, set_gradient_checkpointing, align_special_tokens, set_model_train, model_zero_grad, check_should_save, update_deepspeed_config_by_hidden_size, hf_deepspeed_save_model, sft_mini_metrics, sft_global_metrics, with_hf_deepspeed_context, load_model, configure_model_gradients, init_processor, init_dataset, train_pipeline, ) from thinkstream.trainer.grpo import ( load_grpo_models, unwrap_model_for_generation, rollout, calc_rewards, build_grpo_inputs, apply_liger_kernel_for_grpo, compute_grpo_loss, grpo_micro_metrics, grpo_global_metrics, prepare_grpo_micro_batches, init_grpo_refs, init_grpo_dataset, timer, REWARD_DICT_KEYS, ) TRAINER_BUILDERS = Registry("TrainerBuilders") @TRAINER_BUILDERS.register(key="sft") @builder def build_hf_deepspeed_train() -> Node: train_nodes: list[Node] = [ deepspeed_init_distributed( ddp_timeout=R.args.train.ddp_timeout(metadata={ARG: Arg(default=1800)}), distributed_state=R.distributed.state, ), report_distributed_runtime(distributed_state=R.distributed.state), set_seed(seed=R.args.train.seed(metadata={ARG: Arg(default=42)})), deepspeed_config_init( hidden_size=None, deepspeed_path=R.args.train.deepspeed(metadata={ARG: Arg(required=True)}), deepspeed_config=R.distributed.deepspeed_config, bsz=R.args.train.per_device_train_batch_size( metadata={ARG: Arg(default=1)} ), grad_acc_steps=R.args.train.gradient_accumulation_steps( metadata={ARG: Arg(default=1)} ), num_processes=get_attribute( value=R.distributed.state, attrs=("num_processes",) ), fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), max_grad_norm=R.args.train.max_grad_norm(metadata={ARG: Arg(default=1.0)}), ), setup_dtype( dtype=R.dtype, bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), ), build_optimizer_kwargs( learning_rate=R.args.train.learning_rate( metadata={ARG: Arg(default=2e-07)} ), adam_beta1=R.args.train.adam_beta1(metadata={ARG: Arg(default=0.9)}), adam_beta2=R.args.train.adam_beta2(metadata={ARG: Arg(default=0.999)}), adam_epsilon=R.args.train.adam_epsilon(metadata={ARG: Arg(default=1e-08)}), optimizer_kwargs=R.optimizer_kwargs, ), load_model( model_name_or_path=R.args.model.name_or_path( metadata={ARG: Arg(required=True)} ), model_cache_dir=R.args.model.cache_dir(metadata={ARG: Arg(default=None)}), bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), model=R.model, model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), deepspeed_config=R.distributed.deepspeed_config, vision_attn_implementation=R.args.model.vision_attn_implementation( metadata={ARG: Arg(default="flash_attention_2")} ), ).add_wrappers( with_hf_deepspeed_context(deepspeed_config=R.distributed.deepspeed_config) ), update_deepspeed_config_by_hidden_size( deepspeed_config=R.distributed.deepspeed_config, hidden_size=get_attribute( value=R.model, attrs=("config", "text_config", "hidden_size") ), ), configure_model_gradients(model=R.model), init_processor( model_name_or_path=R.args.model.name_or_path( metadata={ARG: Arg(required=True)} ), model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), processor=R.processor, ), init_dataset( processor=R.processor, model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), train_dataset=R.train_dataset, data_collator=R.data_collator, data_dataset_use=R.args.data.dataset_use( metadata={ARG: Arg(required=True)} ), data_flatten=R.args.data.flatten(metadata={ARG: Arg(default=False)}), data_packing=R.args.data.packing(metadata={ARG: Arg(default=False)}), data_base_interval=R.args.data.base_interval( metadata={ARG: Arg(default=2)} ), data_max_pixels=R.args.data.max_pixels(metadata={ARG: Arg(default=50176)}), data_min_pixels=R.args.data.min_pixels(metadata={ARG: Arg(default=784)}), data_video_max_frames=R.args.data.video_max_frames( metadata={ARG: Arg(default=8)} ), data_video_min_frames=R.args.data.video_min_frames( metadata={ARG: Arg(default=4)} ), data_video_max_pixels=R.args.data.video_max_pixels( metadata={ARG: Arg(default=100352)} ), data_video_min_pixels=R.args.data.video_min_pixels( metadata={ARG: Arg(default=50176)} ), data_video_fps=R.args.data.video_fps(metadata={ARG: Arg(default=2)}), model_max_length=R.args.model.max_length( metadata={ARG: Arg(default=16384)} ), vocab_size=get_attribute( value=R.model, attrs=("config", "text_config", "vocab_size") ), ), apply_liger_kernel(model=R.model), align_special_tokens( model=R.model, tokenizer=get_attribute(value=R.processor, attrs=("tokenizer",)), ), free_memory(), prepare_distributed_dataloader( dataset=R.train_dataset, data_collator=R.data_collator, bsz=R.args.train.per_device_train_batch_size( metadata={ARG: Arg(default=1)} ), num_workers=R.args.train.dataloader.num_workers( metadata={ARG: Arg(default=0)} ), pin_memory=R.args.train.dataloader.pin_memory( metadata={ARG: Arg(default=True)} ), persistent_workers=R.args.train.dataloader.persistent_workers( metadata={ARG: Arg(default=False)} ), drop_last=R.args.train.dataloader.drop_last( metadata={ARG: Arg(default=False)} ), prefetch_factor=R.args.train.dataloader.prefetch_factor( metadata={ARG: Arg(default=None)} ), process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), dataloader=R.train_dataloader, distributed_state=R.distributed.state, seed=R.args.train.seed(metadata={ARG: Arg(default=42)}), ), init_training_state( arg_max_steps=R.args.train.max_steps(metadata={ARG: Arg(default=-1)}), arg_num_train_epochs=R.args.train.num_train_epochs( metadata={ARG: Arg(required=True, type=float)} ), grad_acc_steps=R.args.train.gradient_accumulation_steps( metadata={ARG: Arg(default=1)} ), train_dataloader=R.train_dataloader, drop_last=R.args.train.dataloader.drop_last( metadata={ARG: Arg(default=False)} ), state_num_train_epochs=R.state.train.num_train_epochs, state_max_steps=R.state.train.max_steps, state_log_history=R.state.train.log_history, state_global_step=R.state.train.global_step, state_epoch_idx=R.state.train.epoch_idx, state_epoch=R.state.train.epoch, control_should_stop_epoch=R.control.stop_epoch, control_should_stop_training=R.control.stop_training, ), create_optimizer( model=R.model, weight_decay=R.args.train.weight_decay(metadata={ARG: Arg(default=0.0)}), optimizer_cls=R.optimizer_cls(metadata={ARG: Arg(default="adamw")}), optimizer_kwargs=R.optimizer_kwargs, optimizer=R.optimizer, ), create_scheduler( lr_scheduler_type=R.args.train.lr_scheduler_type( metadata={ARG: Arg(default="cosine")} ), lr_scheduler_kwargs=R.args.train.lr_scheduler_kwargs( metadata={ARG: Arg(default_factory=lambda: {})} ), optimizer=R.optimizer, state_max_steps=R.state.train.max_steps, warmup_ratio=R.args.train.warmup_ratio(metadata={ARG: Arg(default=0.0)}), lr_scheduler=R.lr_scheduler, ), set_gradient_checkpointing( model=R.model, grad_ckpt_kwargs=R.args.train.gradient_checkpointing_kwargs( metadata={ARG: Arg(default=None)} ), ), set_model_train(model=R.model), deepspeed_initialize( model=R.model, optimizer=R.optimizer, model_for_training=R.model_for_training, deepspeed_config=R.distributed.deepspeed_config, lr_scheduler=R.lr_scheduler, ), model_zero_grad(model_for_training=R.model_for_training), init_progress( progress=R.progress, state_max_steps=R.state.train.max_steps, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), ), epoch_loop( nodes=[ dataloader_set_epoch( dataloader=R.train_dataloader, state_epoch_idx=R.state.train.epoch_idx, ), dataloader_loop( mini_step_nodes=[ deepspeed_set_grad_acc_boundary( model_for_training=R.model_for_training, step_should_sync_grad=R.step.should_sync_grad, ), prepare_inputs( step_inputs=R.step.micro.inputs, device=get_attribute( value=R.distributed.state, attrs=("device",) ), dtype=R.dtype, ), compute_loss( step_inputs=R.step.micro.inputs, model_for_training=R.model_for_training, step_loss=R.step.micro.loss, ), collect_metrics( current_metrics=sft_mini_metrics( step_loss=R.step.micro.loss ), step_metrics_history=R.step.metrics_history, ), clean_step_inputs(step_inputs=R.step.micro.inputs), empty_cache( empty_steps=R.args.train.torch_empty_cache_steps( metadata={ARG: Arg(default=1)} ), state_global_step=R.state.train.global_step, ), deepspeed_backward( step_loss=R.step.micro.loss, model_for_training=R.model_for_training, step_current_gas=R.step.current_gas, ), ], global_step_nodes=[ deepspeed_step( model_for_training=R.model_for_training ).add_wrappers( deepspeed_with_grad_acc_boundary( step_should_sync_grad=True, reset_boundary_to=False, model_for_training=R.model_for_training, ) ), collect_metrics( current_metrics=sft_global_metrics( model_for_training=R.model_for_training, optimizer=R.optimizer, ), step_metrics_history=R.step.metrics_history, ), reduce_and_log_metrics( metric_defs={ "loss": "mean", "grad_norm": "latest", "learning_rate": "latest", }, step_metrics_history=R.step.metrics_history, state_log_history=R.state.train.log_history, state_global_step=R.state.train.global_step, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), device=get_attribute( value=R.distributed.state, attrs=("device",) ), ), update_progress( progress=R.progress, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), state_global_step=R.state.train.global_step, ), hf_deepspeed_save_model( model_for_training=R.model_for_training, model=R.model, processor=R.processor, output_dir=R.args.train.output_dir( metadata={ARG: Arg(required=True)} ), process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), state_global_step=R.state.train.global_step, state_log_history=R.state.train.log_history, ).add_wrappers( check_should_save( save_steps=R.args.train.save_steps( metadata={ARG: Arg(default=1000)} ), state_global_step=R.state.train.global_step, ) ), ], dataloader=R.train_dataloader, step=R.step, step_inputs=R.step.micro.inputs, step_current_gas=R.step.current_gas, step_should_sync_grad=R.step.should_sync_grad, state_global_step=R.state.train.global_step, control_should_stop_epoch=R.control.stop_epoch, control_should_stop_training=R.control.stop_training, state_max_steps=R.state.train.max_steps, grad_acc_steps=R.args.train.gradient_accumulation_steps( metadata={ARG: Arg(default=1)} ), ), ], state_num_train_epochs=R.state.train.num_train_epochs, state_epoch_idx=R.state.train.epoch_idx, control_should_stop_training=R.control.stop_training, ), hf_deepspeed_save_model( model_for_training=R.model_for_training, model=R.model, processor=R.processor, output_dir=R.args.train.output_dir(metadata={ARG: Arg(required=True)}), process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), state_global_step=R.state.train.global_step, state_log_history=R.state.train.log_history, ), destroy_progress( progress=R.progress, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), ), ] return train_pipeline(nodes=train_nodes) @TRAINER_BUILDERS.register(key="grpo") @builder def build_grpo_train() -> Node: grpo_nodes: list[Node] = [ deepspeed_init_distributed( ddp_timeout=R.args.train.ddp_timeout(metadata={ARG: Arg(default=1800)}), distributed_state=R.distributed.state, ), set_seed(seed=R.args.train.seed(metadata={ARG: Arg(default=42)})), init_grpo_refs(inference_engine=R.inference_engine), deepspeed_config_init( hidden_size=None, deepspeed_path=R.args.train.deepspeed(metadata={ARG: Arg(required=True)}), deepspeed_config=R.distributed.deepspeed_config, bsz=R.args.train.per_device_train_batch_size( metadata={ARG: Arg(default=1)} ), grad_acc_steps=R.args.train.gradient_accumulation_steps( metadata={ARG: Arg(default=1)} ), num_processes=get_attribute( value=R.distributed.state, attrs=("num_processes",) ), fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), max_grad_norm=R.args.train.max_grad_norm(metadata={ARG: Arg(default=1.0)}), ), setup_dtype( dtype=R.dtype, bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), ), build_optimizer_kwargs( learning_rate=R.args.train.learning_rate( metadata={ARG: Arg(default=2e-07)} ), adam_beta1=R.args.train.adam_beta1(metadata={ARG: Arg(default=0.9)}), adam_beta2=R.args.train.adam_beta2(metadata={ARG: Arg(default=0.999)}), adam_epsilon=R.args.train.adam_epsilon(metadata={ARG: Arg(default=1e-08)}), optimizer_kwargs=R.optimizer_kwargs, ), load_grpo_models( model_name_or_path=R.args.model.name_or_path( metadata={ARG: Arg(required=True)} ), model_cache_dir=R.args.model.cache_dir(metadata={ARG: Arg(default=None)}), bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), reference_model=R.reference_model, model=R.model, model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), model_for_generation=R.model_for_generation, deepspeed_config=R.distributed.deepspeed_config, ), update_deepspeed_config_by_hidden_size( deepspeed_config=R.distributed.deepspeed_config, hidden_size=get_attribute( value=R.model, attrs=("config", "text_config", "hidden_size") ), ), init_processor( model_name_or_path=R.args.model.name_or_path( metadata={ARG: Arg(required=True)} ), model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), processor=R.processor, ), init_grpo_dataset( processor=R.processor, train_dataset=R.train_dataset, data_collator=R.data_collator, data_dataset_use=R.args.data.dataset_use( metadata={ARG: Arg(required=True)} ), rollout_min_pixels=R.args.train.rollout_min_pixels( metadata={ARG: Arg(default=DEFAULT_INFERENCE_MIN_PIXELS)} ), rollout_max_pixels=R.args.train.rollout_max_pixels( metadata={ARG: Arg(default=DEFAULT_INFERENCE_MAX_PIXELS)} ), rollout_fpc=R.args.train.rollout_fpc(metadata={ARG: Arg(default=2.0)}), rollout_max_chunks=R.args.train.rollout_max_chunks( metadata={ARG: Arg(default=DEFAULT_MAX_CHUNKS)} ), model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), ), apply_liger_kernel_for_grpo( model=R.model, reference_model=R.reference_model, model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), ), align_special_tokens( model=R.model, tokenizer=get_attribute(value=R.processor, attrs=("tokenizer",)), ), free_memory(), prepare_distributed_dataloader( dataset=R.train_dataset, data_collator=R.data_collator, bsz=R.args.train.per_device_train_batch_size( metadata={ARG: Arg(default=1)} ), num_workers=R.args.train.dataloader.num_workers( metadata={ARG: Arg(default=0)} ), pin_memory=R.args.train.dataloader.pin_memory( metadata={ARG: Arg(default=True)} ), persistent_workers=R.args.train.dataloader.persistent_workers( metadata={ARG: Arg(default=False)} ), drop_last=R.args.train.dataloader.drop_last( metadata={ARG: Arg(default=False)} ), prefetch_factor=R.args.train.dataloader.prefetch_factor( metadata={ARG: Arg(default=None)} ), process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), dataloader=R.train_dataloader, distributed_state=R.distributed.state, seed=R.args.train.seed(metadata={ARG: Arg(default=42)}), ), init_training_state( arg_max_steps=R.args.train.max_steps(metadata={ARG: Arg(default=-1)}), arg_num_train_epochs=R.args.train.num_train_epochs( metadata={ARG: Arg(required=True, type=float)} ), grad_acc_steps=R.args.train.gradient_accumulation_steps( metadata={ARG: Arg(default=1)} ), train_dataloader=R.train_dataloader, drop_last=R.args.train.dataloader.drop_last( metadata={ARG: Arg(default=False)} ), state_num_train_epochs=R.state.train.num_train_epochs, state_max_steps=R.state.train.max_steps, state_log_history=R.state.train.log_history, state_global_step=R.state.train.global_step, state_epoch_idx=R.state.train.epoch_idx, state_epoch=R.state.train.epoch, control_should_stop_epoch=R.control.stop_epoch, control_should_stop_training=R.control.stop_training, ), create_optimizer( model=R.model, weight_decay=R.args.train.weight_decay(metadata={ARG: Arg(default=0.0)}), optimizer_cls=R.optimizer_cls(metadata={ARG: Arg(default="adamw")}), optimizer_kwargs=R.optimizer_kwargs, optimizer=R.optimizer, ), create_scheduler( lr_scheduler_type=R.args.train.lr_scheduler_type( metadata={ARG: Arg(default="cosine")} ), lr_scheduler_kwargs=R.args.train.lr_scheduler_kwargs( metadata={ARG: Arg(default_factory=lambda: {})} ), optimizer=R.optimizer, state_max_steps=R.state.train.max_steps, warmup_ratio=R.args.train.warmup_ratio(metadata={ARG: Arg(default=0.0)}), lr_scheduler=R.lr_scheduler, ), set_gradient_checkpointing( model=R.model, grad_ckpt_kwargs=R.args.train.gradient_checkpointing_kwargs( metadata={ARG: Arg(default=None)} ), ), set_model_train(model=R.model), deepspeed_initialize( model=R.model, optimizer=R.optimizer, model_for_training=R.model_for_training, deepspeed_config=R.distributed.deepspeed_config, lr_scheduler=R.lr_scheduler, ), model_zero_grad(model_for_training=R.model_for_training), init_progress( progress=R.progress, state_max_steps=R.state.train.max_steps, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), ), epoch_loop( nodes=[ dataloader_set_epoch( dataloader=R.train_dataloader, state_epoch_idx=R.state.train.epoch_idx, ), dataloader_loop_with_micro_steps( mini_step_nodes=[ rollout( step_inputs=R.step.micro.inputs, model_for_generation=R.model_for_generation, processor=R.processor, tokenizer=get_attribute( value=R.processor, attrs=("tokenizer",) ), group_size=R.args.train.group_size( metadata={ARG: Arg(default=8)} ), rollout_data=R.step.rollout_data, inference_engine=R.inference_engine, model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), rollout_max_new_tokens=R.args.train.rollout_max_new_tokens( metadata={ARG: Arg(default=30)} ), rollout_max_think_tokens=R.args.train.rollout_max_think_tokens( metadata={ARG: Arg(default=20)} ), rollout_temperature=R.args.train.rollout_temperature( metadata={ARG: Arg(default=1.0)} ), rollout_top_k=R.args.train.rollout_top_k( metadata={ARG: Arg(default=50)} ), rollout_top_p=R.args.train.rollout_top_p( metadata={ARG: Arg(default=0.95)} ), rollout_fpc=R.args.train.rollout_fpc( metadata={ARG: Arg(default=2.0)} ), rollout_max_chunks=R.args.train.rollout_max_chunks( metadata={ARG: Arg(default=DEFAULT_MAX_CHUNKS)} ), rollout_min_pixels=R.args.train.rollout_min_pixels( metadata={ ARG: Arg(default=DEFAULT_INFERENCE_MIN_PIXELS) } ), rollout_max_pixels=R.args.train.rollout_max_pixels( metadata={ ARG: Arg(default=DEFAULT_INFERENCE_MAX_PIXELS) } ), ).add_wrappers( timer(name="Sync + Rollout"), unwrap_model_for_generation( model_for_training=R.model_for_training, inference_engine=R.inference_engine, model_for_generation=R.model_for_generation, device=get_attribute( value=R.distributed.state, attrs=("device",) ), state_global_step=R.state.train.global_step, rollout_last_sync_step=R.state.grpo.rollout_last_sync_step, ), ), calc_rewards( rollout_data=R.step.rollout_data, rewards=R.step.rewards, rewards_dict=R.step.rewards_dict, group_size=R.args.train.group_size( metadata={ARG: Arg(default=8)} ), tokenizer=get_attribute( value=R.processor, attrs=("tokenizer",) ), time_reward_window=R.args.train.time_reward_window( metadata={ARG: Arg(default=5)} ), time_reward_slack=R.args.train.time_reward_slack( metadata={ARG: Arg(default=3.0)} ), rollout_max_think_tokens=R.args.train.rollout_max_think_tokens( metadata={ARG: Arg(default=20)} ), ), calc_grpo_advantages( rewards=R.step.rewards, group_size=R.args.train.group_size( metadata={ARG: Arg(default=8)} ), advantages=R.step.advantages, ), prepare_grpo_micro_batches( advantages=R.step.advantages, rewards=R.step.rewards, rewards_dict=R.step.rewards_dict, micro_batch_size=R.args.train.micro_batch_size( metadata={ARG: Arg(default=4)} ), group_size=R.args.train.group_size( metadata={ARG: Arg(default=8)} ), step_advantages=R.step.micro.advantages, step_micro_rewards=R.step.micro.rewards, step_micro_rewards_dict=R.step.micro.rewards_dict, step_micro_items=R.step.micro.items, step_micro_batches=R.step.micro_batches, ), ], micro_step_nodes=[ deepspeed_set_grad_acc_boundary( model_for_training=R.model_for_training, step_should_sync_grad=R.step.should_sync_grad, ), build_grpo_inputs( step_micro_items=R.step.micro.items, step_micro_inputs=R.step.micro.inputs, rollout_data=R.step.rollout_data, processor=R.processor, tokenizer=get_attribute( value=R.processor, attrs=("tokenizer",) ), model_type=R.args.model.model_type( metadata={ARG: Arg(default="qwen2.5vl")} ), rollout_fpc=R.args.train.rollout_fpc( metadata={ARG: Arg(default=2.0)} ), ), prepare_inputs( step_inputs=R.step.micro.inputs, device=get_attribute( value=R.distributed.state, attrs=("device",) ), dtype=R.dtype, ), compute_grpo_loss( step_micro_inputs=R.step.micro.inputs, step_advantages=R.step.micro.advantages, model_for_training=R.model_for_training, reference_model=R.reference_model, step_loss=R.step.micro.loss, beta=R.args.train.beta(metadata={ARG: Arg(default=0.001)}), device=get_attribute( value=R.distributed.state, attrs=("device",) ), ).add_wrappers(timer(name="Compute GRPO Loss")), clean_step_inputs(step_inputs=R.step.micro.inputs), empty_cache( empty_steps=R.args.train.torch_empty_cache_steps( metadata={ARG: Arg(default=1)} ), state_global_step=R.state.train.global_step, ), deepspeed_backward( step_loss=R.step.micro.loss, model_for_training=R.model_for_training, step_current_gas=R.step.current_gas, ).add_wrappers(timer(name="Backward")), collect_metrics( current_metrics=grpo_micro_metrics( step_loss=R.step.micro.loss, step_micro_items=R.step.micro.items, rollout_data=R.step.rollout_data, tokenizer=get_attribute( value=R.processor, attrs=("tokenizer",) ), ), step_metrics_history=R.step.metrics_history, ), ], global_step_nodes=[ deepspeed_step( model_for_training=R.model_for_training ).add_wrappers( deepspeed_with_grad_acc_boundary( step_should_sync_grad=True, reset_boundary_to=False, model_for_training=R.model_for_training, ) ), collect_metrics( current_metrics=grpo_global_metrics( model_for_training=R.model_for_training, optimizer=R.optimizer, rewards=R.step.rewards, rewards_dict=R.step.rewards_dict, group_size=R.args.train.group_size( metadata={ARG: Arg(default=8)} ), ), step_metrics_history=R.step.metrics_history, ), reduce_and_log_metrics( metric_defs={ "loss": "mean", "reward_mean": "mean", "reward_var": "mean", "avg_think_len": "mean", "grad_norm": "latest", "learning_rate": "latest", **{ f"reward_{k}_mean": "mean" for k in REWARD_DICT_KEYS }, }, step_metrics_history=R.step.metrics_history, state_log_history=R.state.train.log_history, state_global_step=R.state.train.global_step, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), device=get_attribute( value=R.distributed.state, attrs=("device",) ), ), update_progress( progress=R.progress, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), state_global_step=R.state.train.global_step, ), hf_deepspeed_save_model( model_for_training=R.model_for_training, model=R.model, processor=R.processor, output_dir=R.args.train.output_dir( metadata={ARG: Arg(required=True)} ), process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), state_global_step=R.state.train.global_step, state_log_history=R.state.train.log_history, ).add_wrappers( check_should_save( save_steps=R.args.train.save_steps( metadata={ARG: Arg(default=1000)} ), state_global_step=R.state.train.global_step, ) ), ], dataloader=R.train_dataloader, step=R.step, step_inputs=R.step.micro.inputs, step_current_gas=R.step.current_gas, step_should_sync_grad=R.step.should_sync_grad, state_global_step=R.state.train.global_step, step_micro_batches=R.step.micro_batches, control_should_stop_epoch=R.control.stop_epoch, control_should_stop_training=R.control.stop_training, state_max_steps=R.state.train.max_steps, grad_acc_steps=R.args.train.gradient_accumulation_steps( metadata={ARG: Arg(default=1)} ), ), ], state_num_train_epochs=R.state.train.num_train_epochs, state_epoch_idx=R.state.train.epoch_idx, control_should_stop_training=R.control.stop_training, ), hf_deepspeed_save_model( model_for_training=R.model_for_training, model=R.model, processor=R.processor, output_dir=R.args.train.output_dir(metadata={ARG: Arg(required=True)}), process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), state_global_step=R.state.train.global_step, state_log_history=R.state.train.log_history, ), destroy_progress( progress=R.progress, process_index=get_attribute( value=R.distributed.state, attrs=("process_index",) ), ), ] return train_pipeline(nodes=grpo_nodes)