File size: 424 Bytes
0990c3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | #!/usr/bin/env bash
set -euo pipefail
export NCCL_ASYNC_ERROR_HANDLING=1
export NCCL_P2P_LEVEL=PHB
export CUDA_DEVICE_MAX_CONNECTIONS=1
export OMP_NUM_THREADS=8
torchrun --standalone --nproc-per-node=7 -m nayara.train \
--config configs/nayara_350m.json \
--data /mnt/nvme/nayara/packed \
--output /mnt/nvme/nayara/checkpoints \
--micro-batch 32 \
--grad-accum 2 \
--max-tokens 3800000000 \
--stop-at 15:15
|