#!/usr/bin/env bash

export NCCL_IB_GID_INDEX=3
export NCCL_IB_SL=3
export NCCL_CHECK_DISABLE=1
export NCCL_P2P_DISABLE=0
export NCCL_IB_DISABLE=0
export NCCL_LL_THRESHOLD=16384
export NCCL_IB_CUDA_SUPPORT=1
export NCCL_SOCKET_IFNAME=bond1
export UCX_NET_DEVICES=bond1
export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
export NCCL_COLLNET_ENABLE=0
export SHARP_COLL_ENABLE_SAT=0
export NCCL_NET_GDR_LEVEL=2
export NCCL_IB_QPS_PER_CONNECTION=4
export NCCL_IB_TC=160
export NCCL_PXN_DISABLE=1

# export CUDA_LAUNCH_BLOCKING=1
# export TORCH_SHOW_CPP_STACKTRACES=1

CONFIG=work_dirs/anymo_stage_3_7B_210e_40gpu_change_audio_dim/anymo_stage_3.py
WORD_DIR=work_dirs/anymo_stage_3_7B_210e_40gpu_change_audio_dim
GPUS=8
NNODES=5
NODE_RANK=$1
PORT=${PORT:-29511}
MASTER_ADDR='29.163.187.25' # new 5*8
# MASTER_ADDR='29.163.185.179' # new-2 5*8
# MASTER_ADDR='29.163.187.4' # 2*8 
torchrun --nproc-per-node $GPUS \
    --nnodes=$NNODES \
    --node_rank=$NODE_RANK \
    --master_addr=$MASTER_ADDR \
    --master_port=$PORT \
    tools/train_deepspeed.py --work-dir $WORD_DIR $CONFIG --launcher pytorch ${@:3}

python gpu.py