#!/bin/bash # Data-parallel training on all GPUs of one or more nodes: # sbatch -A -N 2 scripts/train_ddp.sh [key=value ...] # data.batch_size is per GPU. CUDA graphs need a single process. #SBATCH -N 1 #SBATCH -C gpu #SBATCH --ntasks-per-node=1 #SBATCH --gpus-per-node=4 #SBATCH -c 64 #SBATCH -t 02:00:00 #SBATCH -o logs/train_ddp_%j.out set -euo pipefail cd "${SLURM_SUBMIT_DIR:-$(pwd)}" CONFIG="${CONFIG:-configs/gamgam.yaml}" export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}" export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}" # Hugging Face downloads (the Delphes samples are 52 GB) go to scratch too. export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}" MASTER_ADDR="$(scontrol show hostnames "${SLURM_JOB_NODELIST}" | head -n 1)" mkdir -p logs srun uv run torchrun \ --nnodes="${SLURM_NNODES}" --nproc_per_node="${GPUS_PER_NODE:-4}" \ --rdzv_backend=c10d --rdzv_endpoint="${MASTER_ADDR}:${MASTER_PORT:-29500}" \ -m gnn4colliders.cli train "${CONFIG}" "$@"