#!/bin/bash # wait for the next checkpoint save, then restart training on the new data mix J=39291 N=$(grep -c "\[ckpt\] saved" logs/real_$J.log) while [ "$(grep -c "\[ckpt\] saved" logs/real_$J.log)" -eq "$N" ]; do sleep 30; done grep "\[ckpt\] saved" logs/real_$J.log | tail -1 sleep 30 # let the old-checkpoint cleanup finish scancel $J sleep 60 # let Slurm release the nodes sbatch --nodelist=ip-10-0-224-196,ip-10-0-227-25,ip-10-0-228-101,ip-10-0-228-213 train_real.sbatch