| #SBATCH --job-name=rv3 | |
| #SBATCH --partition=batch | |
| #SBATCH --nodes=1 | |
| #SBATCH --ntasks=1 | |
| #SBATCH --cpus-per-task=8 | |
| #SBATCH --mem=64G | |
| #SBATCH --gres=gpu:h200:1 | |
| # dataset_QA lives on sof1-cephfs; only these zones mount it. | |
| #SBATCH --constraint=zone-sof1|zone-gcp-eu1 | |
| #SBATCH --time=00:50:00 | |
| #SBATCH --output=logs/slurm/rv3-%x-%A_%a.out | |
| #SBATCH --error=logs/slurm/rv3-%x-%A_%a.err | |
| # | |
| # One array task = one row of a job list from make_jobs.py. | |
| # | |
| # recon_variants_v3/make_jobs.py --phase train > recon_variants_v3/jobs_train.tsv | |
| # sbatch --array=1-$(wc -l < recon_variants_v3/jobs_train.tsv)%16 \ | |
| # recon_variants_v3/array.sbatch recon_variants_v3/jobs_train.tsv | |
| # | |
| # The row index IS the array index, so a failed row is re-run by resubmitting that | |
| # index. | |
| # | |
| # --time=00:50:00 is sized to the measured T1 baseline (14 min median, 20 min worst | |
| # over 30 C0 models) scaled up for `-r 1` - v3 trains at the native 1080 short side | |
| # rather than 3DGS's default 1600-wide downscale, which is ~1.4x the pixels - NOT to | |
| # the registry's 2h: on this cluster backfill only places a job in a gap it fits | |
| # inside, and a 2h request pends where a 50-minute one starts immediately. | |
| # | |
| # Override per phase; the spread across phases is large: | |
| # --time=00:20:00 the T0/T2/T3/T4 undertraining rows (50-1000 iterations) | |
| # --time=00:50:00 the T1 rows | |
| # --time=01:15:00 the proxy rows: a D10 build at V4 renders 48 bands x 36 views | |
| # BEFORE its 30k training starts | |
| # --time=00:25:00 compress (MesonGS runs training-free here, ~5 min) | |
| set -euo pipefail | |
| ROOT="${DATASET_QA_ROOT:-${SLURM_SUBMIT_DIR:-$PWD}}" | |
| JOBS="${1:?usage: sbatch array.sbatch <jobs.tsv>}" | |
| [ -f "$JOBS" ] || { echo "no such job list: $JOBS" >&2; exit 1; } | |
| row="$(sed -n "${SLURM_ARRAY_TASK_ID:-1}p" "$JOBS")" | |
| [ -n "$row" ] || { echo "array index ${SLURM_ARRAY_TASK_ID:-1} past the end of $JOBS" >&2; exit 1; } | |
| IFS=$'\t' read -r object config_id <<<"$row" | |
| nvidia-smi --query-gpu=name --format=csv,noheader | head -1 | |
| echo "task ${SLURM_ARRAY_TASK_ID:-1}: $object $config_id" | |
| exec "$ROOT/recon_variants_v3/run_row.sh" "$object" "$config_id" | |