| #!/usr/bin/env bash |
|
|
| export NCCL_IB_GID_INDEX=3 |
| export NCCL_IB_SL=3 |
| export NCCL_CHECK_DISABLE=1 |
| export NCCL_P2P_DISABLE=0 |
| export NCCL_IB_DISABLE=0 |
| export NCCL_LL_THRESHOLD=16384 |
| export NCCL_IB_CUDA_SUPPORT=1 |
| export NCCL_SOCKET_IFNAME=bond1 |
| export UCX_NET_DEVICES=bond1 |
| export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6 |
| export NCCL_COLLNET_ENABLE=0 |
| export SHARP_COLL_ENABLE_SAT=0 |
| export NCCL_NET_GDR_LEVEL=2 |
| export NCCL_IB_QPS_PER_CONNECTION=4 |
| export NCCL_IB_TC=160 |
| export NCCL_PXN_DISABLE=1 |
|
|
| |
| |
|
|
| CONFIG=configs/anymo_stage_2.py |
| WORD_DIR=work_dirs/anymo_stage_2_7B_120e_40gpu |
| GPUS=8 |
| NNODES=5 |
| NODE_RANK=$1 |
| PORT=${PORT:-29511} |
| MASTER_ADDR='29.163.185.20' |
| |
| PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ |
| python -m torch.distributed.launch \ |
| --nnodes=$NNODES \ |
| --node_rank=$NODE_RANK \ |
| --master_addr=$MASTER_ADDR \ |
| --nproc_per_node=$GPUS \ |
| --master_port=$PORT \ |
| $(dirname "$0")/train.py \ |
| --work-dir $WORD_DIR \ |
| $CONFIG \ |
| --launcher pytorch ${@:3} |
|
|
| python gpu.py |
|
|
