nayara-v1 / source /scripts /launch_train.sh

Upload folder using huggingface_hub

0990c3f verified 4 months ago

424 Bytes

#!/usr/bin/env bash
set -euo pipefail

export NCCL_ASYNC_ERROR_HANDLING=1
export NCCL_P2P_LEVEL=PHB
export CUDA_DEVICE_MAX_CONNECTIONS=1
export OMP_NUM_THREADS=8

torchrun --standalone --nproc-per-node=7 -m nayara.train \
  --config configs/nayara_350m.json \
  --data /mnt/nvme/nayara/packed \
  --output /mnt/nvme/nayara/checkpoints \
  --micro-batch 32 \
  --grad-accum 2 \
  --max-tokens 3800000000 \
  --stop-at 15:15

	#!/usr/bin/env bash
	set -euo pipefail

	export NCCL_ASYNC_ERROR_HANDLING=1
	export NCCL_P2P_LEVEL=PHB
	export CUDA_DEVICE_MAX_CONNECTIONS=1
	export OMP_NUM_THREADS=8

	torchrun --standalone --nproc-per-node=7 -m nayara.train \
	--config configs/nayara_350m.json \
	--data /mnt/nvme/nayara/packed \
	--output /mnt/nvme/nayara/checkpoints \
	--micro-batch 32 \
	--grad-accum 2 \
	--max-tokens 3800000000 \
	--stop-at 15:15