array.sbatch

#!/bin/bash
#SBATCH --job-name=rv3
#SBATCH --partition=batch
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=8
#SBATCH --mem=64G
#SBATCH --gres=gpu:h200:1
# dataset_QA lives on sof1-cephfs; only these zones mount it.
#SBATCH --constraint=zone-sof1|zone-gcp-eu1
#SBATCH --time=00:50:00
#SBATCH --output=logs/slurm/rv3-%x-%A_%a.out
#SBATCH --error=logs/slurm/rv3-%x-%A_%a.err
#
# One array task = one row of a job list from make_jobs.py.
#
#   recon_variants_v3/make_jobs.py --phase train > recon_variants_v3/jobs_train.tsv
#   sbatch --array=1-$(wc -l < recon_variants_v3/jobs_train.tsv)%16 \
#          recon_variants_v3/array.sbatch recon_variants_v3/jobs_train.tsv
#
# The row index IS the array index, so a failed row is re-run by resubmitting that
# index.
#
# --time=00:50:00 is sized to the measured T1 baseline (14 min median, 20 min worst
# over 30 C0 models) scaled up for `-r 1` - v3 trains at the native 1080 short side
# rather than 3DGS's default 1600-wide downscale, which is ~1.4x the pixels - NOT to
# the registry's 2h: on this cluster backfill only places a job in a gap it fits
# inside, and a 2h request pends where a 50-minute one starts immediately.
#
# Override per phase; the spread across phases is large:
#   --time=00:20:00   the T0/T2/T3/T4 undertraining rows (50-1000 iterations)
#   --time=00:50:00   the T1 rows
#   --time=01:15:00   the proxy rows: a D10 build at V4 renders 48 bands x 36 views
#                     BEFORE its 30k training starts
#   --time=00:25:00   compress (MesonGS runs training-free here, ~5 min)
set -euo pipefail

ROOT="${DATASET_QA_ROOT:-${SLURM_SUBMIT_DIR:-$PWD}}"
JOBS="${1:?usage: sbatch array.sbatch <jobs.tsv>}"
[ -f "$JOBS" ] || { echo "no such job list: $JOBS" >&2; exit 1; }

row="$(sed -n "${SLURM_ARRAY_TASK_ID:-1}p" "$JOBS")"
[ -n "$row" ] || { echo "array index ${SLURM_ARRAY_TASK_ID:-1} past the end of $JOBS" >&2; exit 1; }
IFS=$'\t' read -r object config_id <<<"$row"

nvidia-smi --query-gpu=name --format=csv,noheader | head -1
echo "task ${SLURM_ARRAY_TASK_ID:-1}: $object $config_id"
exec "$ROOT/recon_variants_v3/run_row.sh" "$object" "$config_id"