Download swap.sh from AjayRangoji/AR: direct link, hf CLI and curl.
- Browser
- Download file 520 Bytes
-
https://huggingface.co/AjayRangoji/AR/resolve/main/swap.sh
- Command line
-
hf download hf://AjayRangoji/AR/swap.sh
-
curl -L -o swap.sh https://huggingface.co/AjayRangoji/AR/resolve/main/swap.sh
520 Bytes
| # wait for the next checkpoint save, then restart training on the new data mix | |
| J=39291 | |
| N=$(grep -c "\[ckpt\] saved" logs/real_$J.log) | |
| while [ "$(grep -c "\[ckpt\] saved" logs/real_$J.log)" -eq "$N" ]; do sleep 30; done | |
| grep "\[ckpt\] saved" logs/real_$J.log | tail -1 | |
| sleep 30 # let the old-checkpoint cleanup finish | |
| scancel $J | |
| sleep 60 # let Slurm release the nodes | |
| sbatch --nodelist=ip-10-0-224-196,ip-10-0-227-25,ip-10-0-228-101,ip-10-0-228-213 train_real.sbatch | |