#!/bin/bash #SBATCH --job-name=frtrain #SBATCH --partition=gpu #SBATCH --gres=gpu:h100:1 #SBATCH --nodelist=hpcgpu03 #SBATCH --cpus-per-task=4 #SBATCH --mem=48G #SBATCH --time=14:00:00 #SBATCH --output=%x-%j.out set -euo pipefail JOB=$HOME/pt7m PT=/storage/scratch/$USER/pt7m V=/storage/scratch/$USER/vocos14 nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader,nounits -l 60 -i ${SLURM_JOB_GPUS:-0} > $V/frtrain.gpu.csv & apptainer exec --nv -B /storage/scratch,$HOME $PT/py311.sif bash $JOB/frgen/frtrain.sh $JOB $PT /storage/scratch/$USER/qwen $V kill %1 2>/dev/null || true awk -F', ' '{use += $1; if ($2 > mem) mem = $2} END {if (NR) print "gpu media", use / NR, "% memoria max", mem, "MiB"}' $V/frtrain.gpu.csv || true