AR / swap.sh
AjayRangoji's picture
Upload AR-4B pretraining codebase, configs, and cluster scripts
2ae6691 verified
Raw History Blame Contribute Delete
520 Bytes
#!/bin/bash
# wait for the next checkpoint save, then restart training on the new data mix
J=39291
N=$(grep -c "\[ckpt\] saved" logs/real_$J.log)
while [ "$(grep -c "\[ckpt\] saved" logs/real_$J.log)" -eq "$N" ]; do sleep 30; done
grep "\[ckpt\] saved" logs/real_$J.log | tail -1
sleep 30 # let the old-checkpoint cleanup finish
scancel $J
sleep 60 # let Slurm release the nodes
sbatch --nodelist=ip-10-0-224-196,ip-10-0-227-25,ip-10-0-228-101,ip-10-0-228-213 train_real.sbatch