| #!/usr/bin/env bash |
|
|
| export NCCL_IB_GID_INDEX=3 |
| export NCCL_IB_SL=3 |
| export NCCL_CHECK_DISABLE=1 |
| export NCCL_P2P_DISABLE=0 |
| export NCCL_IB_DISABLE=0 |
| export NCCL_LL_THRESHOLD=16384 |
| export NCCL_IB_CUDA_SUPPORT=1 |
| export NCCL_SOCKET_IFNAME=bond1 |
| export UCX_NET_DEVICES=bond1 |
| export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6 |
| export NCCL_COLLNET_ENABLE=0 |
| export SHARP_COLL_ENABLE_SAT=0 |
| export NCCL_NET_GDR_LEVEL=2 |
| export NCCL_IB_QPS_PER_CONNECTION=4 |
| export NCCL_IB_TC=160 |
| export NCCL_PXN_DISABLE=1 |
|
|
| |
| |
|
|
| CONFIG=work_dirs/anymo_stage_3_7B_210e_40gpu_change_audio_dim/anymo_stage_3.py |
| WORD_DIR=work_dirs/anymo_stage_3_7B_210e_40gpu_change_audio_dim |
| GPUS=8 |
| NNODES=5 |
| NODE_RANK=$1 |
| PORT=${PORT:-29511} |
| MASTER_ADDR='29.163.187.25' |
| |
| |
| torchrun --nproc-per-node $GPUS \ |
| --nnodes=$NNODES \ |
| --node_rank=$NODE_RANK \ |
| --master_addr=$MASTER_ADDR \ |
| --master_port=$PORT \ |
| tools/train_deepspeed.py --work-dir $WORD_DIR $CONFIG --launcher pytorch ${@:3} |
|
|
| python gpu.py |
|
|
