code/train.sbatch

#!/bin/bash
#SBATCH --account=reformo
#SBATCH --partition=booster
#SBATCH --nodes=1
#SBATCH --gres=gpu:4
#SBATCH --ntasks-per-node=4
#SBATCH --cpus-per-task=16
#SBATCH --time=01:00:00
#SBATCH --job-name=m2-cont-large-smoke
#SBATCH --output=/e/scratch/reformo/schuhmann1_moss/out/m2_600m_ladder/logs/continuous_large_%j.out
#SBATCH --error=/e/scratch/reformo/schuhmann1_moss/out/m2_600m_ladder/logs/continuous_large_%j.err
set -euo pipefail
PLAN=$(realpath "$1")
SC=/e/scratch/reformo/schuhmann1_moss
CODE=/e/home/jusers/schuhmann1/jupiter/m2_600m_tts_training/continuous_large_talker_v1
source "$SC/code/fastgen.sh"
export HF_HOME="$SC/hfcache"
export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 TOKENIZERS_PARALLELISM=false
export PYTHONPATH="$CODE:/e/home/jusers/schuhmann1/jupiter/m2_600m_tts_training/cascade_v3_timed:$SC/code/small_tts:$SC/code/vc_env_pkgs:$PYTHONPATH"
export OMP_NUM_THREADS=4 PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
export MASTER_PORT=$((21000 + SLURM_JOB_ID % 19000))
export M2_INVOCATION_SUFFIX="${SLURM_RESTART_COUNT:-}"
export M2_CONTINUOUS_PLAN="$PLAN"
WORLD=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["world_size"])' "$PLAN")
RANKS_PER_NODE=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["ranks_per_node"])' "$PLAN")

srun --ntasks="$WORLD" --ntasks-per-node="$RANKS_PER_NODE" --gpus-per-task=1 \
  --kill-on-bad-exit=1 --cpu-bind=none bash -c '
  cd /e/home/jusers/schuhmann1/jupiter/m2_600m_tts_training/continuous_large_talker_v1
  python3 -u continuous_train.py "$M2_CONTINUOUS_PLAN"
  RC=$?
  echo "M2_CONTINUOUS_TASK RANK=$SLURM_PROCID EXIT=$RC"
  exit $RC
'