| { |
| "_comment": [ |
| "Fallback for 40 GB-class GPUs or shops standardised on DeepSpeed. Only the sharding", |
| "backend changes; the recipe (LR schedule, clipping, loss normalisation, packing) still", |
| "comes from train_sft_qwen3.py, so run it with --skip-checks and replace the FSDP block", |
| "with deepspeed.initialize, or drive it through accelerate's DeepSpeed plugin.", |
| "bf16.enabled with fp32 optimizer states reproduces jmp p=f32,c=bfloat16.", |
| "gradient_clipping is DISABLED here because the script clips explicitly before step().", |
| "Optimizer/scheduler are declared 'none' so the script's AdamW + LambdaLR stay in charge." |
| ], |
| "train_micro_batch_size_per_gpu": 1, |
| "gradient_accumulation_steps": 8, |
| "gradient_clipping": 0.0, |
| "steps_per_print": 1, |
| "wall_clock_breakdown": false, |
| "bf16": { "enabled": true }, |
| "fp16": { "enabled": false }, |
| "zero_optimization": { |
| "stage": 3, |
| "overlap_comm": true, |
| "contiguous_gradients": true, |
| "reduce_bucket_size": 67108864, |
| "stage3_prefetch_bucket_size": 67108864, |
| "stage3_param_persistence_threshold": 65536, |
| "stage3_max_live_parameters": 1000000000, |
| "stage3_max_reuse_distance": 1000000000, |
| "stage3_gather_16bit_weights_on_model_save": true, |
| "offload_optimizer": { "device": "none" }, |
| "offload_param": { "device": "none" } |
| }, |
| "activation_checkpointing": { |
| "partition_activations": false, |
| "cpu_checkpointing": false, |
| "contiguous_memory_optimization": false, |
| "number_checkpoints": 36, |
| "synchronize_checkpoint_boundary": false, |
| "profile": false |
| }, |
| "_offload_note": "On 8x40 GB set offload_optimizer.device='cpu' with pin_memory=true: that frees ~12 GB/GPU at a 20-40% throughput cost and does not change the math." |
| } |
|
|
