GNN4Colliders / scripts /train_ddp.sh
ho22joshua's picture
Claude Opus 5.5
feat: JEPA pretraining of the object transformer
9b40da9
Raw History Blame Contribute Delete
1.01 kB
#!/bin/bash
# Data-parallel training on all GPUs of one or more nodes:
# sbatch -A <account> -N 2 scripts/train_ddp.sh [key=value ...]
# data.batch_size is per GPU. CUDA graphs need a single process.
#SBATCH -N 1
#SBATCH -C gpu
#SBATCH --ntasks-per-node=1
#SBATCH --gpus-per-node=4
#SBATCH -c 64
#SBATCH -t 02:00:00
#SBATCH -o logs/train_ddp_%j.out
set -euo pipefail
cd "${SLURM_SUBMIT_DIR:-$(pwd)}"
CONFIG="${CONFIG:-configs/gamgam.yaml}"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
# Hugging Face downloads (the Delphes samples are 52 GB) go to scratch too.
export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
MASTER_ADDR="$(scontrol show hostnames "${SLURM_JOB_NODELIST}" | head -n 1)"
mkdir -p logs
srun uv run torchrun \
--nnodes="${SLURM_NNODES}" --nproc_per_node="${GPUS_PER_NODE:-4}" \
--rdzv_backend=c10d --rdzv_endpoint="${MASTER_ADDR}:${MASTER_PORT:-29500}" \
-m gnn4colliders.cli train "${CONFIG}" "$@"