File size: 1,012 Bytes
0cef53c
 
2f04f23
0cef53c
 
 
 
 
 
 
 
 
 
 
 
 
 
9b40da9
 
0cef53c
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#!/bin/bash
# Data-parallel training on all GPUs of one or more nodes:
#   sbatch -A <account> -N 2 scripts/train_ddp.sh [key=value ...]
# data.batch_size is per GPU. CUDA graphs need a single process.
#SBATCH -N 1
#SBATCH -C gpu
#SBATCH --ntasks-per-node=1
#SBATCH --gpus-per-node=4
#SBATCH -c 64
#SBATCH -t 02:00:00
#SBATCH -o logs/train_ddp_%j.out

set -euo pipefail
cd "${SLURM_SUBMIT_DIR:-$(pwd)}"
CONFIG="${CONFIG:-configs/gamgam.yaml}"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
# Hugging Face downloads (the Delphes samples are 52 GB) go to scratch too.
export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
MASTER_ADDR="$(scontrol show hostnames "${SLURM_JOB_NODELIST}" | head -n 1)"
mkdir -p logs
srun uv run torchrun \
  --nnodes="${SLURM_NNODES}" --nproc_per_node="${GPUS_PER_NODE:-4}" \
  --rdzv_backend=c10d --rdzv_endpoint="${MASTER_ADDR}:${MASTER_PORT:-29500}" \
  -m gnn4colliders.cli train "${CONFIG}" "$@"