Download ThinkStream/thinkstream/trainer/builder.py from JohnCage/stream-tbstars: direct link, hf CLI and curl.
- Browser
- Download file 41.5 kB
-
https://huggingface.co/JohnCage/stream-tbstars/resolve/main/ThinkStream/thinkstream/trainer/builder.py
- Command line
-
hf download hf://JohnCage/stream-tbstars/ThinkStream/thinkstream/trainer/builder.py
-
curl -L -o builder.py https://huggingface.co/JohnCage/stream-tbstars/resolve/main/ThinkStream/thinkstream/trainer/builder.py
41.5 kB
| from slyme.utils.registry import Registry | |
| from slyme.context import ARG, Arg, R | |
| from slyme.node import Node | |
| from slyme.builder import builder | |
| from thinkstream.trainer.expressions import get_attribute | |
| from thinkstream.data.stream_data_processor import ( | |
| DEFAULT_INFERENCE_MAX_PIXELS, | |
| DEFAULT_INFERENCE_MIN_PIXELS, | |
| DEFAULT_MAX_CHUNKS, | |
| ) | |
| from deepslyme.node.common import ( | |
| set_seed, | |
| free_memory, | |
| empty_cache_by_step as empty_cache, | |
| prepare_distributed_dataloader, | |
| prepare_inputs, | |
| clean_step_inputs, | |
| setup_dtype, | |
| create_optimizer, | |
| create_scheduler, | |
| init_progress, | |
| update_progress, | |
| destroy_progress, | |
| dataloader_loop, | |
| dataloader_loop_with_micro_steps, | |
| ) | |
| from deepslyme.node.integration.deepspeed import ( | |
| deepspeed_init_distributed, | |
| deepspeed_config_init, | |
| deepspeed_backward, | |
| deepspeed_step, | |
| deepspeed_initialize, | |
| deepspeed_set_grad_acc_boundary, | |
| deepspeed_with_grad_acc_boundary, | |
| ) | |
| from deepslyme.node.integration.liger_kernel import apply_liger_kernel | |
| from deepslyme.node.sft import ( | |
| init_training_state, | |
| epoch_loop, | |
| dataloader_set_epoch, | |
| compute_loss, | |
| ) | |
| from deepslyme.node.metric import collect_metrics, reduce_and_log_metrics | |
| from deepslyme.node.rl.grpo import calc_grpo_advantages | |
| from thinkstream.trainer.sft import ( | |
| report_distributed_runtime, | |
| build_optimizer_kwargs, | |
| set_gradient_checkpointing, | |
| align_special_tokens, | |
| set_model_train, | |
| model_zero_grad, | |
| check_should_save, | |
| update_deepspeed_config_by_hidden_size, | |
| hf_deepspeed_save_model, | |
| sft_mini_metrics, | |
| sft_global_metrics, | |
| with_hf_deepspeed_context, | |
| load_model, | |
| configure_model_gradients, | |
| init_processor, | |
| init_dataset, | |
| train_pipeline, | |
| ) | |
| from thinkstream.trainer.grpo import ( | |
| load_grpo_models, | |
| unwrap_model_for_generation, | |
| rollout, | |
| calc_rewards, | |
| build_grpo_inputs, | |
| apply_liger_kernel_for_grpo, | |
| compute_grpo_loss, | |
| grpo_micro_metrics, | |
| grpo_global_metrics, | |
| prepare_grpo_micro_batches, | |
| init_grpo_refs, | |
| init_grpo_dataset, | |
| timer, | |
| REWARD_DICT_KEYS, | |
| ) | |
| TRAINER_BUILDERS = Registry("TrainerBuilders") | |
| def build_hf_deepspeed_train() -> Node: | |
| train_nodes: list[Node] = [ | |
| deepspeed_init_distributed( | |
| ddp_timeout=R.args.train.ddp_timeout(metadata={ARG: Arg(default=1800)}), | |
| distributed_state=R.distributed.state, | |
| ), | |
| report_distributed_runtime(distributed_state=R.distributed.state), | |
| set_seed(seed=R.args.train.seed(metadata={ARG: Arg(default=42)})), | |
| deepspeed_config_init( | |
| hidden_size=None, | |
| deepspeed_path=R.args.train.deepspeed(metadata={ARG: Arg(required=True)}), | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| bsz=R.args.train.per_device_train_batch_size( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| grad_acc_steps=R.args.train.gradient_accumulation_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| num_processes=get_attribute( | |
| value=R.distributed.state, attrs=("num_processes",) | |
| ), | |
| fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), | |
| bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), | |
| max_grad_norm=R.args.train.max_grad_norm(metadata={ARG: Arg(default=1.0)}), | |
| ), | |
| setup_dtype( | |
| dtype=R.dtype, | |
| bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), | |
| fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), | |
| ), | |
| build_optimizer_kwargs( | |
| learning_rate=R.args.train.learning_rate( | |
| metadata={ARG: Arg(default=2e-07)} | |
| ), | |
| adam_beta1=R.args.train.adam_beta1(metadata={ARG: Arg(default=0.9)}), | |
| adam_beta2=R.args.train.adam_beta2(metadata={ARG: Arg(default=0.999)}), | |
| adam_epsilon=R.args.train.adam_epsilon(metadata={ARG: Arg(default=1e-08)}), | |
| optimizer_kwargs=R.optimizer_kwargs, | |
| ), | |
| load_model( | |
| model_name_or_path=R.args.model.name_or_path( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| model_cache_dir=R.args.model.cache_dir(metadata={ARG: Arg(default=None)}), | |
| bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), | |
| model=R.model, | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| vision_attn_implementation=R.args.model.vision_attn_implementation( | |
| metadata={ARG: Arg(default="flash_attention_2")} | |
| ), | |
| ).add_wrappers( | |
| with_hf_deepspeed_context(deepspeed_config=R.distributed.deepspeed_config) | |
| ), | |
| update_deepspeed_config_by_hidden_size( | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| hidden_size=get_attribute( | |
| value=R.model, attrs=("config", "text_config", "hidden_size") | |
| ), | |
| ), | |
| configure_model_gradients(model=R.model), | |
| init_processor( | |
| model_name_or_path=R.args.model.name_or_path( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| processor=R.processor, | |
| ), | |
| init_dataset( | |
| processor=R.processor, | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| train_dataset=R.train_dataset, | |
| data_collator=R.data_collator, | |
| data_dataset_use=R.args.data.dataset_use( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| data_flatten=R.args.data.flatten(metadata={ARG: Arg(default=False)}), | |
| data_packing=R.args.data.packing(metadata={ARG: Arg(default=False)}), | |
| data_base_interval=R.args.data.base_interval( | |
| metadata={ARG: Arg(default=2)} | |
| ), | |
| data_max_pixels=R.args.data.max_pixels(metadata={ARG: Arg(default=50176)}), | |
| data_min_pixels=R.args.data.min_pixels(metadata={ARG: Arg(default=784)}), | |
| data_video_max_frames=R.args.data.video_max_frames( | |
| metadata={ARG: Arg(default=8)} | |
| ), | |
| data_video_min_frames=R.args.data.video_min_frames( | |
| metadata={ARG: Arg(default=4)} | |
| ), | |
| data_video_max_pixels=R.args.data.video_max_pixels( | |
| metadata={ARG: Arg(default=100352)} | |
| ), | |
| data_video_min_pixels=R.args.data.video_min_pixels( | |
| metadata={ARG: Arg(default=50176)} | |
| ), | |
| data_video_fps=R.args.data.video_fps(metadata={ARG: Arg(default=2)}), | |
| model_max_length=R.args.model.max_length( | |
| metadata={ARG: Arg(default=16384)} | |
| ), | |
| vocab_size=get_attribute( | |
| value=R.model, attrs=("config", "text_config", "vocab_size") | |
| ), | |
| ), | |
| apply_liger_kernel(model=R.model), | |
| align_special_tokens( | |
| model=R.model, | |
| tokenizer=get_attribute(value=R.processor, attrs=("tokenizer",)), | |
| ), | |
| free_memory(), | |
| prepare_distributed_dataloader( | |
| dataset=R.train_dataset, | |
| data_collator=R.data_collator, | |
| bsz=R.args.train.per_device_train_batch_size( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| num_workers=R.args.train.dataloader.num_workers( | |
| metadata={ARG: Arg(default=0)} | |
| ), | |
| pin_memory=R.args.train.dataloader.pin_memory( | |
| metadata={ARG: Arg(default=True)} | |
| ), | |
| persistent_workers=R.args.train.dataloader.persistent_workers( | |
| metadata={ARG: Arg(default=False)} | |
| ), | |
| drop_last=R.args.train.dataloader.drop_last( | |
| metadata={ARG: Arg(default=False)} | |
| ), | |
| prefetch_factor=R.args.train.dataloader.prefetch_factor( | |
| metadata={ARG: Arg(default=None)} | |
| ), | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| dataloader=R.train_dataloader, | |
| distributed_state=R.distributed.state, | |
| seed=R.args.train.seed(metadata={ARG: Arg(default=42)}), | |
| ), | |
| init_training_state( | |
| arg_max_steps=R.args.train.max_steps(metadata={ARG: Arg(default=-1)}), | |
| arg_num_train_epochs=R.args.train.num_train_epochs( | |
| metadata={ARG: Arg(required=True, type=float)} | |
| ), | |
| grad_acc_steps=R.args.train.gradient_accumulation_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| train_dataloader=R.train_dataloader, | |
| drop_last=R.args.train.dataloader.drop_last( | |
| metadata={ARG: Arg(default=False)} | |
| ), | |
| state_num_train_epochs=R.state.train.num_train_epochs, | |
| state_max_steps=R.state.train.max_steps, | |
| state_log_history=R.state.train.log_history, | |
| state_global_step=R.state.train.global_step, | |
| state_epoch_idx=R.state.train.epoch_idx, | |
| state_epoch=R.state.train.epoch, | |
| control_should_stop_epoch=R.control.stop_epoch, | |
| control_should_stop_training=R.control.stop_training, | |
| ), | |
| create_optimizer( | |
| model=R.model, | |
| weight_decay=R.args.train.weight_decay(metadata={ARG: Arg(default=0.0)}), | |
| optimizer_cls=R.optimizer_cls(metadata={ARG: Arg(default="adamw")}), | |
| optimizer_kwargs=R.optimizer_kwargs, | |
| optimizer=R.optimizer, | |
| ), | |
| create_scheduler( | |
| lr_scheduler_type=R.args.train.lr_scheduler_type( | |
| metadata={ARG: Arg(default="cosine")} | |
| ), | |
| lr_scheduler_kwargs=R.args.train.lr_scheduler_kwargs( | |
| metadata={ARG: Arg(default_factory=lambda: {})} | |
| ), | |
| optimizer=R.optimizer, | |
| state_max_steps=R.state.train.max_steps, | |
| warmup_ratio=R.args.train.warmup_ratio(metadata={ARG: Arg(default=0.0)}), | |
| lr_scheduler=R.lr_scheduler, | |
| ), | |
| set_gradient_checkpointing( | |
| model=R.model, | |
| grad_ckpt_kwargs=R.args.train.gradient_checkpointing_kwargs( | |
| metadata={ARG: Arg(default=None)} | |
| ), | |
| ), | |
| set_model_train(model=R.model), | |
| deepspeed_initialize( | |
| model=R.model, | |
| optimizer=R.optimizer, | |
| model_for_training=R.model_for_training, | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| lr_scheduler=R.lr_scheduler, | |
| ), | |
| model_zero_grad(model_for_training=R.model_for_training), | |
| init_progress( | |
| progress=R.progress, | |
| state_max_steps=R.state.train.max_steps, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| ), | |
| epoch_loop( | |
| nodes=[ | |
| dataloader_set_epoch( | |
| dataloader=R.train_dataloader, | |
| state_epoch_idx=R.state.train.epoch_idx, | |
| ), | |
| dataloader_loop( | |
| mini_step_nodes=[ | |
| deepspeed_set_grad_acc_boundary( | |
| model_for_training=R.model_for_training, | |
| step_should_sync_grad=R.step.should_sync_grad, | |
| ), | |
| prepare_inputs( | |
| step_inputs=R.step.micro.inputs, | |
| device=get_attribute( | |
| value=R.distributed.state, attrs=("device",) | |
| ), | |
| dtype=R.dtype, | |
| ), | |
| compute_loss( | |
| step_inputs=R.step.micro.inputs, | |
| model_for_training=R.model_for_training, | |
| step_loss=R.step.micro.loss, | |
| ), | |
| collect_metrics( | |
| current_metrics=sft_mini_metrics( | |
| step_loss=R.step.micro.loss | |
| ), | |
| step_metrics_history=R.step.metrics_history, | |
| ), | |
| clean_step_inputs(step_inputs=R.step.micro.inputs), | |
| empty_cache( | |
| empty_steps=R.args.train.torch_empty_cache_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| ), | |
| deepspeed_backward( | |
| step_loss=R.step.micro.loss, | |
| model_for_training=R.model_for_training, | |
| step_current_gas=R.step.current_gas, | |
| ), | |
| ], | |
| global_step_nodes=[ | |
| deepspeed_step( | |
| model_for_training=R.model_for_training | |
| ).add_wrappers( | |
| deepspeed_with_grad_acc_boundary( | |
| step_should_sync_grad=True, | |
| reset_boundary_to=False, | |
| model_for_training=R.model_for_training, | |
| ) | |
| ), | |
| collect_metrics( | |
| current_metrics=sft_global_metrics( | |
| model_for_training=R.model_for_training, | |
| optimizer=R.optimizer, | |
| ), | |
| step_metrics_history=R.step.metrics_history, | |
| ), | |
| reduce_and_log_metrics( | |
| metric_defs={ | |
| "loss": "mean", | |
| "grad_norm": "latest", | |
| "learning_rate": "latest", | |
| }, | |
| step_metrics_history=R.step.metrics_history, | |
| state_log_history=R.state.train.log_history, | |
| state_global_step=R.state.train.global_step, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| device=get_attribute( | |
| value=R.distributed.state, attrs=("device",) | |
| ), | |
| ), | |
| update_progress( | |
| progress=R.progress, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| ), | |
| hf_deepspeed_save_model( | |
| model_for_training=R.model_for_training, | |
| model=R.model, | |
| processor=R.processor, | |
| output_dir=R.args.train.output_dir( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| state_log_history=R.state.train.log_history, | |
| ).add_wrappers( | |
| check_should_save( | |
| save_steps=R.args.train.save_steps( | |
| metadata={ARG: Arg(default=1000)} | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| ) | |
| ), | |
| ], | |
| dataloader=R.train_dataloader, | |
| step=R.step, | |
| step_inputs=R.step.micro.inputs, | |
| step_current_gas=R.step.current_gas, | |
| step_should_sync_grad=R.step.should_sync_grad, | |
| state_global_step=R.state.train.global_step, | |
| control_should_stop_epoch=R.control.stop_epoch, | |
| control_should_stop_training=R.control.stop_training, | |
| state_max_steps=R.state.train.max_steps, | |
| grad_acc_steps=R.args.train.gradient_accumulation_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| ), | |
| ], | |
| state_num_train_epochs=R.state.train.num_train_epochs, | |
| state_epoch_idx=R.state.train.epoch_idx, | |
| control_should_stop_training=R.control.stop_training, | |
| ), | |
| hf_deepspeed_save_model( | |
| model_for_training=R.model_for_training, | |
| model=R.model, | |
| processor=R.processor, | |
| output_dir=R.args.train.output_dir(metadata={ARG: Arg(required=True)}), | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| state_log_history=R.state.train.log_history, | |
| ), | |
| destroy_progress( | |
| progress=R.progress, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| ), | |
| ] | |
| return train_pipeline(nodes=train_nodes) | |
| def build_grpo_train() -> Node: | |
| grpo_nodes: list[Node] = [ | |
| deepspeed_init_distributed( | |
| ddp_timeout=R.args.train.ddp_timeout(metadata={ARG: Arg(default=1800)}), | |
| distributed_state=R.distributed.state, | |
| ), | |
| set_seed(seed=R.args.train.seed(metadata={ARG: Arg(default=42)})), | |
| init_grpo_refs(inference_engine=R.inference_engine), | |
| deepspeed_config_init( | |
| hidden_size=None, | |
| deepspeed_path=R.args.train.deepspeed(metadata={ARG: Arg(required=True)}), | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| bsz=R.args.train.per_device_train_batch_size( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| grad_acc_steps=R.args.train.gradient_accumulation_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| num_processes=get_attribute( | |
| value=R.distributed.state, attrs=("num_processes",) | |
| ), | |
| fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), | |
| bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), | |
| max_grad_norm=R.args.train.max_grad_norm(metadata={ARG: Arg(default=1.0)}), | |
| ), | |
| setup_dtype( | |
| dtype=R.dtype, | |
| bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), | |
| fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}), | |
| ), | |
| build_optimizer_kwargs( | |
| learning_rate=R.args.train.learning_rate( | |
| metadata={ARG: Arg(default=2e-07)} | |
| ), | |
| adam_beta1=R.args.train.adam_beta1(metadata={ARG: Arg(default=0.9)}), | |
| adam_beta2=R.args.train.adam_beta2(metadata={ARG: Arg(default=0.999)}), | |
| adam_epsilon=R.args.train.adam_epsilon(metadata={ARG: Arg(default=1e-08)}), | |
| optimizer_kwargs=R.optimizer_kwargs, | |
| ), | |
| load_grpo_models( | |
| model_name_or_path=R.args.model.name_or_path( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| model_cache_dir=R.args.model.cache_dir(metadata={ARG: Arg(default=None)}), | |
| bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}), | |
| reference_model=R.reference_model, | |
| model=R.model, | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| model_for_generation=R.model_for_generation, | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| ), | |
| update_deepspeed_config_by_hidden_size( | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| hidden_size=get_attribute( | |
| value=R.model, attrs=("config", "text_config", "hidden_size") | |
| ), | |
| ), | |
| init_processor( | |
| model_name_or_path=R.args.model.name_or_path( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| processor=R.processor, | |
| ), | |
| init_grpo_dataset( | |
| processor=R.processor, | |
| train_dataset=R.train_dataset, | |
| data_collator=R.data_collator, | |
| data_dataset_use=R.args.data.dataset_use( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| rollout_min_pixels=R.args.train.rollout_min_pixels( | |
| metadata={ARG: Arg(default=DEFAULT_INFERENCE_MIN_PIXELS)} | |
| ), | |
| rollout_max_pixels=R.args.train.rollout_max_pixels( | |
| metadata={ARG: Arg(default=DEFAULT_INFERENCE_MAX_PIXELS)} | |
| ), | |
| rollout_fpc=R.args.train.rollout_fpc(metadata={ARG: Arg(default=2.0)}), | |
| rollout_max_chunks=R.args.train.rollout_max_chunks( | |
| metadata={ARG: Arg(default=DEFAULT_MAX_CHUNKS)} | |
| ), | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| ), | |
| apply_liger_kernel_for_grpo( | |
| model=R.model, | |
| reference_model=R.reference_model, | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| ), | |
| align_special_tokens( | |
| model=R.model, | |
| tokenizer=get_attribute(value=R.processor, attrs=("tokenizer",)), | |
| ), | |
| free_memory(), | |
| prepare_distributed_dataloader( | |
| dataset=R.train_dataset, | |
| data_collator=R.data_collator, | |
| bsz=R.args.train.per_device_train_batch_size( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| num_workers=R.args.train.dataloader.num_workers( | |
| metadata={ARG: Arg(default=0)} | |
| ), | |
| pin_memory=R.args.train.dataloader.pin_memory( | |
| metadata={ARG: Arg(default=True)} | |
| ), | |
| persistent_workers=R.args.train.dataloader.persistent_workers( | |
| metadata={ARG: Arg(default=False)} | |
| ), | |
| drop_last=R.args.train.dataloader.drop_last( | |
| metadata={ARG: Arg(default=False)} | |
| ), | |
| prefetch_factor=R.args.train.dataloader.prefetch_factor( | |
| metadata={ARG: Arg(default=None)} | |
| ), | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| dataloader=R.train_dataloader, | |
| distributed_state=R.distributed.state, | |
| seed=R.args.train.seed(metadata={ARG: Arg(default=42)}), | |
| ), | |
| init_training_state( | |
| arg_max_steps=R.args.train.max_steps(metadata={ARG: Arg(default=-1)}), | |
| arg_num_train_epochs=R.args.train.num_train_epochs( | |
| metadata={ARG: Arg(required=True, type=float)} | |
| ), | |
| grad_acc_steps=R.args.train.gradient_accumulation_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| train_dataloader=R.train_dataloader, | |
| drop_last=R.args.train.dataloader.drop_last( | |
| metadata={ARG: Arg(default=False)} | |
| ), | |
| state_num_train_epochs=R.state.train.num_train_epochs, | |
| state_max_steps=R.state.train.max_steps, | |
| state_log_history=R.state.train.log_history, | |
| state_global_step=R.state.train.global_step, | |
| state_epoch_idx=R.state.train.epoch_idx, | |
| state_epoch=R.state.train.epoch, | |
| control_should_stop_epoch=R.control.stop_epoch, | |
| control_should_stop_training=R.control.stop_training, | |
| ), | |
| create_optimizer( | |
| model=R.model, | |
| weight_decay=R.args.train.weight_decay(metadata={ARG: Arg(default=0.0)}), | |
| optimizer_cls=R.optimizer_cls(metadata={ARG: Arg(default="adamw")}), | |
| optimizer_kwargs=R.optimizer_kwargs, | |
| optimizer=R.optimizer, | |
| ), | |
| create_scheduler( | |
| lr_scheduler_type=R.args.train.lr_scheduler_type( | |
| metadata={ARG: Arg(default="cosine")} | |
| ), | |
| lr_scheduler_kwargs=R.args.train.lr_scheduler_kwargs( | |
| metadata={ARG: Arg(default_factory=lambda: {})} | |
| ), | |
| optimizer=R.optimizer, | |
| state_max_steps=R.state.train.max_steps, | |
| warmup_ratio=R.args.train.warmup_ratio(metadata={ARG: Arg(default=0.0)}), | |
| lr_scheduler=R.lr_scheduler, | |
| ), | |
| set_gradient_checkpointing( | |
| model=R.model, | |
| grad_ckpt_kwargs=R.args.train.gradient_checkpointing_kwargs( | |
| metadata={ARG: Arg(default=None)} | |
| ), | |
| ), | |
| set_model_train(model=R.model), | |
| deepspeed_initialize( | |
| model=R.model, | |
| optimizer=R.optimizer, | |
| model_for_training=R.model_for_training, | |
| deepspeed_config=R.distributed.deepspeed_config, | |
| lr_scheduler=R.lr_scheduler, | |
| ), | |
| model_zero_grad(model_for_training=R.model_for_training), | |
| init_progress( | |
| progress=R.progress, | |
| state_max_steps=R.state.train.max_steps, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| ), | |
| epoch_loop( | |
| nodes=[ | |
| dataloader_set_epoch( | |
| dataloader=R.train_dataloader, | |
| state_epoch_idx=R.state.train.epoch_idx, | |
| ), | |
| dataloader_loop_with_micro_steps( | |
| mini_step_nodes=[ | |
| rollout( | |
| step_inputs=R.step.micro.inputs, | |
| model_for_generation=R.model_for_generation, | |
| processor=R.processor, | |
| tokenizer=get_attribute( | |
| value=R.processor, attrs=("tokenizer",) | |
| ), | |
| group_size=R.args.train.group_size( | |
| metadata={ARG: Arg(default=8)} | |
| ), | |
| rollout_data=R.step.rollout_data, | |
| inference_engine=R.inference_engine, | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| rollout_max_new_tokens=R.args.train.rollout_max_new_tokens( | |
| metadata={ARG: Arg(default=30)} | |
| ), | |
| rollout_max_think_tokens=R.args.train.rollout_max_think_tokens( | |
| metadata={ARG: Arg(default=20)} | |
| ), | |
| rollout_temperature=R.args.train.rollout_temperature( | |
| metadata={ARG: Arg(default=1.0)} | |
| ), | |
| rollout_top_k=R.args.train.rollout_top_k( | |
| metadata={ARG: Arg(default=50)} | |
| ), | |
| rollout_top_p=R.args.train.rollout_top_p( | |
| metadata={ARG: Arg(default=0.95)} | |
| ), | |
| rollout_fpc=R.args.train.rollout_fpc( | |
| metadata={ARG: Arg(default=2.0)} | |
| ), | |
| rollout_max_chunks=R.args.train.rollout_max_chunks( | |
| metadata={ARG: Arg(default=DEFAULT_MAX_CHUNKS)} | |
| ), | |
| rollout_min_pixels=R.args.train.rollout_min_pixels( | |
| metadata={ | |
| ARG: Arg(default=DEFAULT_INFERENCE_MIN_PIXELS) | |
| } | |
| ), | |
| rollout_max_pixels=R.args.train.rollout_max_pixels( | |
| metadata={ | |
| ARG: Arg(default=DEFAULT_INFERENCE_MAX_PIXELS) | |
| } | |
| ), | |
| ).add_wrappers( | |
| timer(name="Sync + Rollout"), | |
| unwrap_model_for_generation( | |
| model_for_training=R.model_for_training, | |
| inference_engine=R.inference_engine, | |
| model_for_generation=R.model_for_generation, | |
| device=get_attribute( | |
| value=R.distributed.state, attrs=("device",) | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| rollout_last_sync_step=R.state.grpo.rollout_last_sync_step, | |
| ), | |
| ), | |
| calc_rewards( | |
| rollout_data=R.step.rollout_data, | |
| rewards=R.step.rewards, | |
| rewards_dict=R.step.rewards_dict, | |
| group_size=R.args.train.group_size( | |
| metadata={ARG: Arg(default=8)} | |
| ), | |
| tokenizer=get_attribute( | |
| value=R.processor, attrs=("tokenizer",) | |
| ), | |
| time_reward_window=R.args.train.time_reward_window( | |
| metadata={ARG: Arg(default=5)} | |
| ), | |
| time_reward_slack=R.args.train.time_reward_slack( | |
| metadata={ARG: Arg(default=3.0)} | |
| ), | |
| rollout_max_think_tokens=R.args.train.rollout_max_think_tokens( | |
| metadata={ARG: Arg(default=20)} | |
| ), | |
| ), | |
| calc_grpo_advantages( | |
| rewards=R.step.rewards, | |
| group_size=R.args.train.group_size( | |
| metadata={ARG: Arg(default=8)} | |
| ), | |
| advantages=R.step.advantages, | |
| ), | |
| prepare_grpo_micro_batches( | |
| advantages=R.step.advantages, | |
| rewards=R.step.rewards, | |
| rewards_dict=R.step.rewards_dict, | |
| micro_batch_size=R.args.train.micro_batch_size( | |
| metadata={ARG: Arg(default=4)} | |
| ), | |
| group_size=R.args.train.group_size( | |
| metadata={ARG: Arg(default=8)} | |
| ), | |
| step_advantages=R.step.micro.advantages, | |
| step_micro_rewards=R.step.micro.rewards, | |
| step_micro_rewards_dict=R.step.micro.rewards_dict, | |
| step_micro_items=R.step.micro.items, | |
| step_micro_batches=R.step.micro_batches, | |
| ), | |
| ], | |
| micro_step_nodes=[ | |
| deepspeed_set_grad_acc_boundary( | |
| model_for_training=R.model_for_training, | |
| step_should_sync_grad=R.step.should_sync_grad, | |
| ), | |
| build_grpo_inputs( | |
| step_micro_items=R.step.micro.items, | |
| step_micro_inputs=R.step.micro.inputs, | |
| rollout_data=R.step.rollout_data, | |
| processor=R.processor, | |
| tokenizer=get_attribute( | |
| value=R.processor, attrs=("tokenizer",) | |
| ), | |
| model_type=R.args.model.model_type( | |
| metadata={ARG: Arg(default="qwen2.5vl")} | |
| ), | |
| rollout_fpc=R.args.train.rollout_fpc( | |
| metadata={ARG: Arg(default=2.0)} | |
| ), | |
| ), | |
| prepare_inputs( | |
| step_inputs=R.step.micro.inputs, | |
| device=get_attribute( | |
| value=R.distributed.state, attrs=("device",) | |
| ), | |
| dtype=R.dtype, | |
| ), | |
| compute_grpo_loss( | |
| step_micro_inputs=R.step.micro.inputs, | |
| step_advantages=R.step.micro.advantages, | |
| model_for_training=R.model_for_training, | |
| reference_model=R.reference_model, | |
| step_loss=R.step.micro.loss, | |
| beta=R.args.train.beta(metadata={ARG: Arg(default=0.001)}), | |
| device=get_attribute( | |
| value=R.distributed.state, attrs=("device",) | |
| ), | |
| ).add_wrappers(timer(name="Compute GRPO Loss")), | |
| clean_step_inputs(step_inputs=R.step.micro.inputs), | |
| empty_cache( | |
| empty_steps=R.args.train.torch_empty_cache_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| ), | |
| deepspeed_backward( | |
| step_loss=R.step.micro.loss, | |
| model_for_training=R.model_for_training, | |
| step_current_gas=R.step.current_gas, | |
| ).add_wrappers(timer(name="Backward")), | |
| collect_metrics( | |
| current_metrics=grpo_micro_metrics( | |
| step_loss=R.step.micro.loss, | |
| step_micro_items=R.step.micro.items, | |
| rollout_data=R.step.rollout_data, | |
| tokenizer=get_attribute( | |
| value=R.processor, attrs=("tokenizer",) | |
| ), | |
| ), | |
| step_metrics_history=R.step.metrics_history, | |
| ), | |
| ], | |
| global_step_nodes=[ | |
| deepspeed_step( | |
| model_for_training=R.model_for_training | |
| ).add_wrappers( | |
| deepspeed_with_grad_acc_boundary( | |
| step_should_sync_grad=True, | |
| reset_boundary_to=False, | |
| model_for_training=R.model_for_training, | |
| ) | |
| ), | |
| collect_metrics( | |
| current_metrics=grpo_global_metrics( | |
| model_for_training=R.model_for_training, | |
| optimizer=R.optimizer, | |
| rewards=R.step.rewards, | |
| rewards_dict=R.step.rewards_dict, | |
| group_size=R.args.train.group_size( | |
| metadata={ARG: Arg(default=8)} | |
| ), | |
| ), | |
| step_metrics_history=R.step.metrics_history, | |
| ), | |
| reduce_and_log_metrics( | |
| metric_defs={ | |
| "loss": "mean", | |
| "reward_mean": "mean", | |
| "reward_var": "mean", | |
| "avg_think_len": "mean", | |
| "grad_norm": "latest", | |
| "learning_rate": "latest", | |
| **{ | |
| f"reward_{k}_mean": "mean" for k in REWARD_DICT_KEYS | |
| }, | |
| }, | |
| step_metrics_history=R.step.metrics_history, | |
| state_log_history=R.state.train.log_history, | |
| state_global_step=R.state.train.global_step, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| device=get_attribute( | |
| value=R.distributed.state, attrs=("device",) | |
| ), | |
| ), | |
| update_progress( | |
| progress=R.progress, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| ), | |
| hf_deepspeed_save_model( | |
| model_for_training=R.model_for_training, | |
| model=R.model, | |
| processor=R.processor, | |
| output_dir=R.args.train.output_dir( | |
| metadata={ARG: Arg(required=True)} | |
| ), | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| state_log_history=R.state.train.log_history, | |
| ).add_wrappers( | |
| check_should_save( | |
| save_steps=R.args.train.save_steps( | |
| metadata={ARG: Arg(default=1000)} | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| ) | |
| ), | |
| ], | |
| dataloader=R.train_dataloader, | |
| step=R.step, | |
| step_inputs=R.step.micro.inputs, | |
| step_current_gas=R.step.current_gas, | |
| step_should_sync_grad=R.step.should_sync_grad, | |
| state_global_step=R.state.train.global_step, | |
| step_micro_batches=R.step.micro_batches, | |
| control_should_stop_epoch=R.control.stop_epoch, | |
| control_should_stop_training=R.control.stop_training, | |
| state_max_steps=R.state.train.max_steps, | |
| grad_acc_steps=R.args.train.gradient_accumulation_steps( | |
| metadata={ARG: Arg(default=1)} | |
| ), | |
| ), | |
| ], | |
| state_num_train_epochs=R.state.train.num_train_epochs, | |
| state_epoch_idx=R.state.train.epoch_idx, | |
| control_should_stop_training=R.control.stop_training, | |
| ), | |
| hf_deepspeed_save_model( | |
| model_for_training=R.model_for_training, | |
| model=R.model, | |
| processor=R.processor, | |
| output_dir=R.args.train.output_dir(metadata={ARG: Arg(required=True)}), | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| state_global_step=R.state.train.global_step, | |
| state_log_history=R.state.train.log_history, | |
| ), | |
| destroy_progress( | |
| progress=R.progress, | |
| process_index=get_attribute( | |
| value=R.distributed.state, attrs=("process_index",) | |
| ), | |
| ), | |
| ] | |
| return train_pipeline(nodes=grpo_nodes) | |