JohnCage's picture
Upload folder using huggingface_hub
8a2fd95 verified
Raw History Blame Contribute Delete
41.5 kB
from slyme.utils.registry import Registry
from slyme.context import ARG, Arg, R
from slyme.node import Node
from slyme.builder import builder
from thinkstream.trainer.expressions import get_attribute
from thinkstream.data.stream_data_processor import (
DEFAULT_INFERENCE_MAX_PIXELS,
DEFAULT_INFERENCE_MIN_PIXELS,
DEFAULT_MAX_CHUNKS,
)
from deepslyme.node.common import (
set_seed,
free_memory,
empty_cache_by_step as empty_cache,
prepare_distributed_dataloader,
prepare_inputs,
clean_step_inputs,
setup_dtype,
create_optimizer,
create_scheduler,
init_progress,
update_progress,
destroy_progress,
dataloader_loop,
dataloader_loop_with_micro_steps,
)
from deepslyme.node.integration.deepspeed import (
deepspeed_init_distributed,
deepspeed_config_init,
deepspeed_backward,
deepspeed_step,
deepspeed_initialize,
deepspeed_set_grad_acc_boundary,
deepspeed_with_grad_acc_boundary,
)
from deepslyme.node.integration.liger_kernel import apply_liger_kernel
from deepslyme.node.sft import (
init_training_state,
epoch_loop,
dataloader_set_epoch,
compute_loss,
)
from deepslyme.node.metric import collect_metrics, reduce_and_log_metrics
from deepslyme.node.rl.grpo import calc_grpo_advantages
from thinkstream.trainer.sft import (
report_distributed_runtime,
build_optimizer_kwargs,
set_gradient_checkpointing,
align_special_tokens,
set_model_train,
model_zero_grad,
check_should_save,
update_deepspeed_config_by_hidden_size,
hf_deepspeed_save_model,
sft_mini_metrics,
sft_global_metrics,
with_hf_deepspeed_context,
load_model,
configure_model_gradients,
init_processor,
init_dataset,
train_pipeline,
)
from thinkstream.trainer.grpo import (
load_grpo_models,
unwrap_model_for_generation,
rollout,
calc_rewards,
build_grpo_inputs,
apply_liger_kernel_for_grpo,
compute_grpo_loss,
grpo_micro_metrics,
grpo_global_metrics,
prepare_grpo_micro_batches,
init_grpo_refs,
init_grpo_dataset,
timer,
REWARD_DICT_KEYS,
)
TRAINER_BUILDERS = Registry("TrainerBuilders")
@TRAINER_BUILDERS.register(key="sft")
@builder
def build_hf_deepspeed_train() -> Node:
train_nodes: list[Node] = [
deepspeed_init_distributed(
ddp_timeout=R.args.train.ddp_timeout(metadata={ARG: Arg(default=1800)}),
distributed_state=R.distributed.state,
),
report_distributed_runtime(distributed_state=R.distributed.state),
set_seed(seed=R.args.train.seed(metadata={ARG: Arg(default=42)})),
deepspeed_config_init(
hidden_size=None,
deepspeed_path=R.args.train.deepspeed(metadata={ARG: Arg(required=True)}),
deepspeed_config=R.distributed.deepspeed_config,
bsz=R.args.train.per_device_train_batch_size(
metadata={ARG: Arg(default=1)}
),
grad_acc_steps=R.args.train.gradient_accumulation_steps(
metadata={ARG: Arg(default=1)}
),
num_processes=get_attribute(
value=R.distributed.state, attrs=("num_processes",)
),
fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}),
bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}),
max_grad_norm=R.args.train.max_grad_norm(metadata={ARG: Arg(default=1.0)}),
),
setup_dtype(
dtype=R.dtype,
bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}),
fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}),
),
build_optimizer_kwargs(
learning_rate=R.args.train.learning_rate(
metadata={ARG: Arg(default=2e-07)}
),
adam_beta1=R.args.train.adam_beta1(metadata={ARG: Arg(default=0.9)}),
adam_beta2=R.args.train.adam_beta2(metadata={ARG: Arg(default=0.999)}),
adam_epsilon=R.args.train.adam_epsilon(metadata={ARG: Arg(default=1e-08)}),
optimizer_kwargs=R.optimizer_kwargs,
),
load_model(
model_name_or_path=R.args.model.name_or_path(
metadata={ARG: Arg(required=True)}
),
model_cache_dir=R.args.model.cache_dir(metadata={ARG: Arg(default=None)}),
bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}),
model=R.model,
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
deepspeed_config=R.distributed.deepspeed_config,
vision_attn_implementation=R.args.model.vision_attn_implementation(
metadata={ARG: Arg(default="flash_attention_2")}
),
).add_wrappers(
with_hf_deepspeed_context(deepspeed_config=R.distributed.deepspeed_config)
),
update_deepspeed_config_by_hidden_size(
deepspeed_config=R.distributed.deepspeed_config,
hidden_size=get_attribute(
value=R.model, attrs=("config", "text_config", "hidden_size")
),
),
configure_model_gradients(model=R.model),
init_processor(
model_name_or_path=R.args.model.name_or_path(
metadata={ARG: Arg(required=True)}
),
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
processor=R.processor,
),
init_dataset(
processor=R.processor,
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
train_dataset=R.train_dataset,
data_collator=R.data_collator,
data_dataset_use=R.args.data.dataset_use(
metadata={ARG: Arg(required=True)}
),
data_flatten=R.args.data.flatten(metadata={ARG: Arg(default=False)}),
data_packing=R.args.data.packing(metadata={ARG: Arg(default=False)}),
data_base_interval=R.args.data.base_interval(
metadata={ARG: Arg(default=2)}
),
data_max_pixels=R.args.data.max_pixels(metadata={ARG: Arg(default=50176)}),
data_min_pixels=R.args.data.min_pixels(metadata={ARG: Arg(default=784)}),
data_video_max_frames=R.args.data.video_max_frames(
metadata={ARG: Arg(default=8)}
),
data_video_min_frames=R.args.data.video_min_frames(
metadata={ARG: Arg(default=4)}
),
data_video_max_pixels=R.args.data.video_max_pixels(
metadata={ARG: Arg(default=100352)}
),
data_video_min_pixels=R.args.data.video_min_pixels(
metadata={ARG: Arg(default=50176)}
),
data_video_fps=R.args.data.video_fps(metadata={ARG: Arg(default=2)}),
model_max_length=R.args.model.max_length(
metadata={ARG: Arg(default=16384)}
),
vocab_size=get_attribute(
value=R.model, attrs=("config", "text_config", "vocab_size")
),
),
apply_liger_kernel(model=R.model),
align_special_tokens(
model=R.model,
tokenizer=get_attribute(value=R.processor, attrs=("tokenizer",)),
),
free_memory(),
prepare_distributed_dataloader(
dataset=R.train_dataset,
data_collator=R.data_collator,
bsz=R.args.train.per_device_train_batch_size(
metadata={ARG: Arg(default=1)}
),
num_workers=R.args.train.dataloader.num_workers(
metadata={ARG: Arg(default=0)}
),
pin_memory=R.args.train.dataloader.pin_memory(
metadata={ARG: Arg(default=True)}
),
persistent_workers=R.args.train.dataloader.persistent_workers(
metadata={ARG: Arg(default=False)}
),
drop_last=R.args.train.dataloader.drop_last(
metadata={ARG: Arg(default=False)}
),
prefetch_factor=R.args.train.dataloader.prefetch_factor(
metadata={ARG: Arg(default=None)}
),
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
dataloader=R.train_dataloader,
distributed_state=R.distributed.state,
seed=R.args.train.seed(metadata={ARG: Arg(default=42)}),
),
init_training_state(
arg_max_steps=R.args.train.max_steps(metadata={ARG: Arg(default=-1)}),
arg_num_train_epochs=R.args.train.num_train_epochs(
metadata={ARG: Arg(required=True, type=float)}
),
grad_acc_steps=R.args.train.gradient_accumulation_steps(
metadata={ARG: Arg(default=1)}
),
train_dataloader=R.train_dataloader,
drop_last=R.args.train.dataloader.drop_last(
metadata={ARG: Arg(default=False)}
),
state_num_train_epochs=R.state.train.num_train_epochs,
state_max_steps=R.state.train.max_steps,
state_log_history=R.state.train.log_history,
state_global_step=R.state.train.global_step,
state_epoch_idx=R.state.train.epoch_idx,
state_epoch=R.state.train.epoch,
control_should_stop_epoch=R.control.stop_epoch,
control_should_stop_training=R.control.stop_training,
),
create_optimizer(
model=R.model,
weight_decay=R.args.train.weight_decay(metadata={ARG: Arg(default=0.0)}),
optimizer_cls=R.optimizer_cls(metadata={ARG: Arg(default="adamw")}),
optimizer_kwargs=R.optimizer_kwargs,
optimizer=R.optimizer,
),
create_scheduler(
lr_scheduler_type=R.args.train.lr_scheduler_type(
metadata={ARG: Arg(default="cosine")}
),
lr_scheduler_kwargs=R.args.train.lr_scheduler_kwargs(
metadata={ARG: Arg(default_factory=lambda: {})}
),
optimizer=R.optimizer,
state_max_steps=R.state.train.max_steps,
warmup_ratio=R.args.train.warmup_ratio(metadata={ARG: Arg(default=0.0)}),
lr_scheduler=R.lr_scheduler,
),
set_gradient_checkpointing(
model=R.model,
grad_ckpt_kwargs=R.args.train.gradient_checkpointing_kwargs(
metadata={ARG: Arg(default=None)}
),
),
set_model_train(model=R.model),
deepspeed_initialize(
model=R.model,
optimizer=R.optimizer,
model_for_training=R.model_for_training,
deepspeed_config=R.distributed.deepspeed_config,
lr_scheduler=R.lr_scheduler,
),
model_zero_grad(model_for_training=R.model_for_training),
init_progress(
progress=R.progress,
state_max_steps=R.state.train.max_steps,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
),
epoch_loop(
nodes=[
dataloader_set_epoch(
dataloader=R.train_dataloader,
state_epoch_idx=R.state.train.epoch_idx,
),
dataloader_loop(
mini_step_nodes=[
deepspeed_set_grad_acc_boundary(
model_for_training=R.model_for_training,
step_should_sync_grad=R.step.should_sync_grad,
),
prepare_inputs(
step_inputs=R.step.micro.inputs,
device=get_attribute(
value=R.distributed.state, attrs=("device",)
),
dtype=R.dtype,
),
compute_loss(
step_inputs=R.step.micro.inputs,
model_for_training=R.model_for_training,
step_loss=R.step.micro.loss,
),
collect_metrics(
current_metrics=sft_mini_metrics(
step_loss=R.step.micro.loss
),
step_metrics_history=R.step.metrics_history,
),
clean_step_inputs(step_inputs=R.step.micro.inputs),
empty_cache(
empty_steps=R.args.train.torch_empty_cache_steps(
metadata={ARG: Arg(default=1)}
),
state_global_step=R.state.train.global_step,
),
deepspeed_backward(
step_loss=R.step.micro.loss,
model_for_training=R.model_for_training,
step_current_gas=R.step.current_gas,
),
],
global_step_nodes=[
deepspeed_step(
model_for_training=R.model_for_training
).add_wrappers(
deepspeed_with_grad_acc_boundary(
step_should_sync_grad=True,
reset_boundary_to=False,
model_for_training=R.model_for_training,
)
),
collect_metrics(
current_metrics=sft_global_metrics(
model_for_training=R.model_for_training,
optimizer=R.optimizer,
),
step_metrics_history=R.step.metrics_history,
),
reduce_and_log_metrics(
metric_defs={
"loss": "mean",
"grad_norm": "latest",
"learning_rate": "latest",
},
step_metrics_history=R.step.metrics_history,
state_log_history=R.state.train.log_history,
state_global_step=R.state.train.global_step,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
device=get_attribute(
value=R.distributed.state, attrs=("device",)
),
),
update_progress(
progress=R.progress,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
state_global_step=R.state.train.global_step,
),
hf_deepspeed_save_model(
model_for_training=R.model_for_training,
model=R.model,
processor=R.processor,
output_dir=R.args.train.output_dir(
metadata={ARG: Arg(required=True)}
),
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
state_global_step=R.state.train.global_step,
state_log_history=R.state.train.log_history,
).add_wrappers(
check_should_save(
save_steps=R.args.train.save_steps(
metadata={ARG: Arg(default=1000)}
),
state_global_step=R.state.train.global_step,
)
),
],
dataloader=R.train_dataloader,
step=R.step,
step_inputs=R.step.micro.inputs,
step_current_gas=R.step.current_gas,
step_should_sync_grad=R.step.should_sync_grad,
state_global_step=R.state.train.global_step,
control_should_stop_epoch=R.control.stop_epoch,
control_should_stop_training=R.control.stop_training,
state_max_steps=R.state.train.max_steps,
grad_acc_steps=R.args.train.gradient_accumulation_steps(
metadata={ARG: Arg(default=1)}
),
),
],
state_num_train_epochs=R.state.train.num_train_epochs,
state_epoch_idx=R.state.train.epoch_idx,
control_should_stop_training=R.control.stop_training,
),
hf_deepspeed_save_model(
model_for_training=R.model_for_training,
model=R.model,
processor=R.processor,
output_dir=R.args.train.output_dir(metadata={ARG: Arg(required=True)}),
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
state_global_step=R.state.train.global_step,
state_log_history=R.state.train.log_history,
),
destroy_progress(
progress=R.progress,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
),
]
return train_pipeline(nodes=train_nodes)
@TRAINER_BUILDERS.register(key="grpo")
@builder
def build_grpo_train() -> Node:
grpo_nodes: list[Node] = [
deepspeed_init_distributed(
ddp_timeout=R.args.train.ddp_timeout(metadata={ARG: Arg(default=1800)}),
distributed_state=R.distributed.state,
),
set_seed(seed=R.args.train.seed(metadata={ARG: Arg(default=42)})),
init_grpo_refs(inference_engine=R.inference_engine),
deepspeed_config_init(
hidden_size=None,
deepspeed_path=R.args.train.deepspeed(metadata={ARG: Arg(required=True)}),
deepspeed_config=R.distributed.deepspeed_config,
bsz=R.args.train.per_device_train_batch_size(
metadata={ARG: Arg(default=1)}
),
grad_acc_steps=R.args.train.gradient_accumulation_steps(
metadata={ARG: Arg(default=1)}
),
num_processes=get_attribute(
value=R.distributed.state, attrs=("num_processes",)
),
fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}),
bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}),
max_grad_norm=R.args.train.max_grad_norm(metadata={ARG: Arg(default=1.0)}),
),
setup_dtype(
dtype=R.dtype,
bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}),
fp16=R.args.train.fp16(metadata={ARG: Arg(default=False)}),
),
build_optimizer_kwargs(
learning_rate=R.args.train.learning_rate(
metadata={ARG: Arg(default=2e-07)}
),
adam_beta1=R.args.train.adam_beta1(metadata={ARG: Arg(default=0.9)}),
adam_beta2=R.args.train.adam_beta2(metadata={ARG: Arg(default=0.999)}),
adam_epsilon=R.args.train.adam_epsilon(metadata={ARG: Arg(default=1e-08)}),
optimizer_kwargs=R.optimizer_kwargs,
),
load_grpo_models(
model_name_or_path=R.args.model.name_or_path(
metadata={ARG: Arg(required=True)}
),
model_cache_dir=R.args.model.cache_dir(metadata={ARG: Arg(default=None)}),
bf16=R.args.train.bf16(metadata={ARG: Arg(default=True)}),
reference_model=R.reference_model,
model=R.model,
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
model_for_generation=R.model_for_generation,
deepspeed_config=R.distributed.deepspeed_config,
),
update_deepspeed_config_by_hidden_size(
deepspeed_config=R.distributed.deepspeed_config,
hidden_size=get_attribute(
value=R.model, attrs=("config", "text_config", "hidden_size")
),
),
init_processor(
model_name_or_path=R.args.model.name_or_path(
metadata={ARG: Arg(required=True)}
),
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
processor=R.processor,
),
init_grpo_dataset(
processor=R.processor,
train_dataset=R.train_dataset,
data_collator=R.data_collator,
data_dataset_use=R.args.data.dataset_use(
metadata={ARG: Arg(required=True)}
),
rollout_min_pixels=R.args.train.rollout_min_pixels(
metadata={ARG: Arg(default=DEFAULT_INFERENCE_MIN_PIXELS)}
),
rollout_max_pixels=R.args.train.rollout_max_pixels(
metadata={ARG: Arg(default=DEFAULT_INFERENCE_MAX_PIXELS)}
),
rollout_fpc=R.args.train.rollout_fpc(metadata={ARG: Arg(default=2.0)}),
rollout_max_chunks=R.args.train.rollout_max_chunks(
metadata={ARG: Arg(default=DEFAULT_MAX_CHUNKS)}
),
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
),
apply_liger_kernel_for_grpo(
model=R.model,
reference_model=R.reference_model,
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
),
align_special_tokens(
model=R.model,
tokenizer=get_attribute(value=R.processor, attrs=("tokenizer",)),
),
free_memory(),
prepare_distributed_dataloader(
dataset=R.train_dataset,
data_collator=R.data_collator,
bsz=R.args.train.per_device_train_batch_size(
metadata={ARG: Arg(default=1)}
),
num_workers=R.args.train.dataloader.num_workers(
metadata={ARG: Arg(default=0)}
),
pin_memory=R.args.train.dataloader.pin_memory(
metadata={ARG: Arg(default=True)}
),
persistent_workers=R.args.train.dataloader.persistent_workers(
metadata={ARG: Arg(default=False)}
),
drop_last=R.args.train.dataloader.drop_last(
metadata={ARG: Arg(default=False)}
),
prefetch_factor=R.args.train.dataloader.prefetch_factor(
metadata={ARG: Arg(default=None)}
),
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
dataloader=R.train_dataloader,
distributed_state=R.distributed.state,
seed=R.args.train.seed(metadata={ARG: Arg(default=42)}),
),
init_training_state(
arg_max_steps=R.args.train.max_steps(metadata={ARG: Arg(default=-1)}),
arg_num_train_epochs=R.args.train.num_train_epochs(
metadata={ARG: Arg(required=True, type=float)}
),
grad_acc_steps=R.args.train.gradient_accumulation_steps(
metadata={ARG: Arg(default=1)}
),
train_dataloader=R.train_dataloader,
drop_last=R.args.train.dataloader.drop_last(
metadata={ARG: Arg(default=False)}
),
state_num_train_epochs=R.state.train.num_train_epochs,
state_max_steps=R.state.train.max_steps,
state_log_history=R.state.train.log_history,
state_global_step=R.state.train.global_step,
state_epoch_idx=R.state.train.epoch_idx,
state_epoch=R.state.train.epoch,
control_should_stop_epoch=R.control.stop_epoch,
control_should_stop_training=R.control.stop_training,
),
create_optimizer(
model=R.model,
weight_decay=R.args.train.weight_decay(metadata={ARG: Arg(default=0.0)}),
optimizer_cls=R.optimizer_cls(metadata={ARG: Arg(default="adamw")}),
optimizer_kwargs=R.optimizer_kwargs,
optimizer=R.optimizer,
),
create_scheduler(
lr_scheduler_type=R.args.train.lr_scheduler_type(
metadata={ARG: Arg(default="cosine")}
),
lr_scheduler_kwargs=R.args.train.lr_scheduler_kwargs(
metadata={ARG: Arg(default_factory=lambda: {})}
),
optimizer=R.optimizer,
state_max_steps=R.state.train.max_steps,
warmup_ratio=R.args.train.warmup_ratio(metadata={ARG: Arg(default=0.0)}),
lr_scheduler=R.lr_scheduler,
),
set_gradient_checkpointing(
model=R.model,
grad_ckpt_kwargs=R.args.train.gradient_checkpointing_kwargs(
metadata={ARG: Arg(default=None)}
),
),
set_model_train(model=R.model),
deepspeed_initialize(
model=R.model,
optimizer=R.optimizer,
model_for_training=R.model_for_training,
deepspeed_config=R.distributed.deepspeed_config,
lr_scheduler=R.lr_scheduler,
),
model_zero_grad(model_for_training=R.model_for_training),
init_progress(
progress=R.progress,
state_max_steps=R.state.train.max_steps,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
),
epoch_loop(
nodes=[
dataloader_set_epoch(
dataloader=R.train_dataloader,
state_epoch_idx=R.state.train.epoch_idx,
),
dataloader_loop_with_micro_steps(
mini_step_nodes=[
rollout(
step_inputs=R.step.micro.inputs,
model_for_generation=R.model_for_generation,
processor=R.processor,
tokenizer=get_attribute(
value=R.processor, attrs=("tokenizer",)
),
group_size=R.args.train.group_size(
metadata={ARG: Arg(default=8)}
),
rollout_data=R.step.rollout_data,
inference_engine=R.inference_engine,
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
rollout_max_new_tokens=R.args.train.rollout_max_new_tokens(
metadata={ARG: Arg(default=30)}
),
rollout_max_think_tokens=R.args.train.rollout_max_think_tokens(
metadata={ARG: Arg(default=20)}
),
rollout_temperature=R.args.train.rollout_temperature(
metadata={ARG: Arg(default=1.0)}
),
rollout_top_k=R.args.train.rollout_top_k(
metadata={ARG: Arg(default=50)}
),
rollout_top_p=R.args.train.rollout_top_p(
metadata={ARG: Arg(default=0.95)}
),
rollout_fpc=R.args.train.rollout_fpc(
metadata={ARG: Arg(default=2.0)}
),
rollout_max_chunks=R.args.train.rollout_max_chunks(
metadata={ARG: Arg(default=DEFAULT_MAX_CHUNKS)}
),
rollout_min_pixels=R.args.train.rollout_min_pixels(
metadata={
ARG: Arg(default=DEFAULT_INFERENCE_MIN_PIXELS)
}
),
rollout_max_pixels=R.args.train.rollout_max_pixels(
metadata={
ARG: Arg(default=DEFAULT_INFERENCE_MAX_PIXELS)
}
),
).add_wrappers(
timer(name="Sync + Rollout"),
unwrap_model_for_generation(
model_for_training=R.model_for_training,
inference_engine=R.inference_engine,
model_for_generation=R.model_for_generation,
device=get_attribute(
value=R.distributed.state, attrs=("device",)
),
state_global_step=R.state.train.global_step,
rollout_last_sync_step=R.state.grpo.rollout_last_sync_step,
),
),
calc_rewards(
rollout_data=R.step.rollout_data,
rewards=R.step.rewards,
rewards_dict=R.step.rewards_dict,
group_size=R.args.train.group_size(
metadata={ARG: Arg(default=8)}
),
tokenizer=get_attribute(
value=R.processor, attrs=("tokenizer",)
),
time_reward_window=R.args.train.time_reward_window(
metadata={ARG: Arg(default=5)}
),
time_reward_slack=R.args.train.time_reward_slack(
metadata={ARG: Arg(default=3.0)}
),
rollout_max_think_tokens=R.args.train.rollout_max_think_tokens(
metadata={ARG: Arg(default=20)}
),
),
calc_grpo_advantages(
rewards=R.step.rewards,
group_size=R.args.train.group_size(
metadata={ARG: Arg(default=8)}
),
advantages=R.step.advantages,
),
prepare_grpo_micro_batches(
advantages=R.step.advantages,
rewards=R.step.rewards,
rewards_dict=R.step.rewards_dict,
micro_batch_size=R.args.train.micro_batch_size(
metadata={ARG: Arg(default=4)}
),
group_size=R.args.train.group_size(
metadata={ARG: Arg(default=8)}
),
step_advantages=R.step.micro.advantages,
step_micro_rewards=R.step.micro.rewards,
step_micro_rewards_dict=R.step.micro.rewards_dict,
step_micro_items=R.step.micro.items,
step_micro_batches=R.step.micro_batches,
),
],
micro_step_nodes=[
deepspeed_set_grad_acc_boundary(
model_for_training=R.model_for_training,
step_should_sync_grad=R.step.should_sync_grad,
),
build_grpo_inputs(
step_micro_items=R.step.micro.items,
step_micro_inputs=R.step.micro.inputs,
rollout_data=R.step.rollout_data,
processor=R.processor,
tokenizer=get_attribute(
value=R.processor, attrs=("tokenizer",)
),
model_type=R.args.model.model_type(
metadata={ARG: Arg(default="qwen2.5vl")}
),
rollout_fpc=R.args.train.rollout_fpc(
metadata={ARG: Arg(default=2.0)}
),
),
prepare_inputs(
step_inputs=R.step.micro.inputs,
device=get_attribute(
value=R.distributed.state, attrs=("device",)
),
dtype=R.dtype,
),
compute_grpo_loss(
step_micro_inputs=R.step.micro.inputs,
step_advantages=R.step.micro.advantages,
model_for_training=R.model_for_training,
reference_model=R.reference_model,
step_loss=R.step.micro.loss,
beta=R.args.train.beta(metadata={ARG: Arg(default=0.001)}),
device=get_attribute(
value=R.distributed.state, attrs=("device",)
),
).add_wrappers(timer(name="Compute GRPO Loss")),
clean_step_inputs(step_inputs=R.step.micro.inputs),
empty_cache(
empty_steps=R.args.train.torch_empty_cache_steps(
metadata={ARG: Arg(default=1)}
),
state_global_step=R.state.train.global_step,
),
deepspeed_backward(
step_loss=R.step.micro.loss,
model_for_training=R.model_for_training,
step_current_gas=R.step.current_gas,
).add_wrappers(timer(name="Backward")),
collect_metrics(
current_metrics=grpo_micro_metrics(
step_loss=R.step.micro.loss,
step_micro_items=R.step.micro.items,
rollout_data=R.step.rollout_data,
tokenizer=get_attribute(
value=R.processor, attrs=("tokenizer",)
),
),
step_metrics_history=R.step.metrics_history,
),
],
global_step_nodes=[
deepspeed_step(
model_for_training=R.model_for_training
).add_wrappers(
deepspeed_with_grad_acc_boundary(
step_should_sync_grad=True,
reset_boundary_to=False,
model_for_training=R.model_for_training,
)
),
collect_metrics(
current_metrics=grpo_global_metrics(
model_for_training=R.model_for_training,
optimizer=R.optimizer,
rewards=R.step.rewards,
rewards_dict=R.step.rewards_dict,
group_size=R.args.train.group_size(
metadata={ARG: Arg(default=8)}
),
),
step_metrics_history=R.step.metrics_history,
),
reduce_and_log_metrics(
metric_defs={
"loss": "mean",
"reward_mean": "mean",
"reward_var": "mean",
"avg_think_len": "mean",
"grad_norm": "latest",
"learning_rate": "latest",
**{
f"reward_{k}_mean": "mean" for k in REWARD_DICT_KEYS
},
},
step_metrics_history=R.step.metrics_history,
state_log_history=R.state.train.log_history,
state_global_step=R.state.train.global_step,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
device=get_attribute(
value=R.distributed.state, attrs=("device",)
),
),
update_progress(
progress=R.progress,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
state_global_step=R.state.train.global_step,
),
hf_deepspeed_save_model(
model_for_training=R.model_for_training,
model=R.model,
processor=R.processor,
output_dir=R.args.train.output_dir(
metadata={ARG: Arg(required=True)}
),
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
state_global_step=R.state.train.global_step,
state_log_history=R.state.train.log_history,
).add_wrappers(
check_should_save(
save_steps=R.args.train.save_steps(
metadata={ARG: Arg(default=1000)}
),
state_global_step=R.state.train.global_step,
)
),
],
dataloader=R.train_dataloader,
step=R.step,
step_inputs=R.step.micro.inputs,
step_current_gas=R.step.current_gas,
step_should_sync_grad=R.step.should_sync_grad,
state_global_step=R.state.train.global_step,
step_micro_batches=R.step.micro_batches,
control_should_stop_epoch=R.control.stop_epoch,
control_should_stop_training=R.control.stop_training,
state_max_steps=R.state.train.max_steps,
grad_acc_steps=R.args.train.gradient_accumulation_steps(
metadata={ARG: Arg(default=1)}
),
),
],
state_num_train_epochs=R.state.train.num_train_epochs,
state_epoch_idx=R.state.train.epoch_idx,
control_should_stop_training=R.control.stop_training,
),
hf_deepspeed_save_model(
model_for_training=R.model_for_training,
model=R.model,
processor=R.processor,
output_dir=R.args.train.output_dir(metadata={ARG: Arg(required=True)}),
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
state_global_step=R.state.train.global_step,
state_log_history=R.state.train.log_history,
),
destroy_progress(
progress=R.progress,
process_index=get_attribute(
value=R.distributed.state, attrs=("process_index",)
),
),
]
return train_pipeline(nodes=grpo_nodes)