prosatts-work / scripts /frgen /frtrain.sbatch
marcosremar2's picture
Checkpoints, banco de risos e scripts
63a4356 verified
Raw History Blame Contribute Delete
745 Bytes
#!/bin/bash
#SBATCH --job-name=frtrain
#SBATCH --partition=gpu
#SBATCH --gres=gpu:h100:1
#SBATCH --nodelist=hpcgpu03
#SBATCH --cpus-per-task=4
#SBATCH --mem=48G
#SBATCH --time=14:00:00
#SBATCH --output=%x-%j.out
set -euo pipefail
JOB=$HOME/pt7m
PT=/storage/scratch/$USER/pt7m
V=/storage/scratch/$USER/vocos14
nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader,nounits -l 60 -i ${SLURM_JOB_GPUS:-0} > $V/frtrain.gpu.csv &
apptainer exec --nv -B /storage/scratch,$HOME $PT/py311.sif bash $JOB/frgen/frtrain.sh $JOB $PT /storage/scratch/$USER/qwen $V
kill %1 2>/dev/null || true
awk -F', ' '{use += $1; if ($2 > mem) mem = $2} END {if (NR) print "gpu media", use / NR, "% memoria max", mem, "MiB"}' $V/frtrain.gpu.csv || true