File size: 1,766 Bytes
58258b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | # Optional: run train_sft_qwen3.py under `accelerate launch` instead of `torchrun`.
# The script builds its own FSDP wrapper, so accelerate is used ONLY as a launcher here
# (it sets RANK/LOCAL_RANK/WORLD_SIZE). These fsdp_config values document the exact
# equivalent settings for anyone who prefers accelerate's Accelerator.prepare().
#
# accelerate launch --config_file accelerate_fsdp_qwen3.yaml train_sft_qwen3.py ...
#
# If you DO switch to Accelerator.prepare(), you must keep:
# * mixed_precision bf16 with fp32 master weights and fp32 gradient reduction
# * FULL_SHARD + transformer-layer wrapping of Qwen3DecoderLayer
# * use_orig_params: true (the weight-decay parameter groups depend on it)
# * accelerator.clip_grad_norm_ AFTER all accumulation micro-steps
# * your own loss normalisation; accelerate's default 1/gradient_accumulation_steps
# is NOT the recipe's per-microbatch token-weighted mean.
compute_environment: LOCAL_MACHINE
distributed_type: FSDP
downcast_bf16: 'no'
machine_rank: 0
main_training_function: main
mixed_precision: bf16
num_machines: 1
num_processes: 8
rdzv_backend: static
same_network: true
use_cpu: false
enable_cpu_affinity: true
fsdp_config:
fsdp_version: 1
fsdp_sharding_strategy: FULL_SHARD
fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
fsdp_transformer_layer_cls_to_wrap: Qwen3DecoderLayer
fsdp_backward_prefetch: BACKWARD_PRE
fsdp_forward_prefetch: false
fsdp_use_orig_params: true
fsdp_cpu_ram_efficient_loading: true
fsdp_sync_module_states: true
fsdp_offload_params: false
fsdp_state_dict_type: SHARDED_STATE_DICT
fsdp_activation_checkpointing: false # the script calls gradient_checkpointing_enable itself
fsdp_reshard_after_forward: true
fsdp_limit_all_gathers: true
|