| #!/usr/bin/env bash | |
| set -euo pipefail | |
| seed71_log=/mloscratch/homes/navasard/sparse/train_logs/p2_1p2b_mixed_seed71_4gpu_resume45250.log | |
| seed72_log=/mloscratch/homes/navasard/sparse/train_logs/p2_1p2b_mixed_seed72_4gpu_resume31000_after_seed71.log | |
| repo=/mloscratch/homes/navasard/sparse/SparseAtt | |
| while ! grep -q "Training complete" "$seed71_log"; do sleep 60; done | |
| if pgrep -f "torchrun.*p2_1p2b_mixed_seed72_4gpu_autoresume_current.json" >/dev/null; then exit 0; fi | |
| cd "$repo" | |
| nohup env USER=navasard LOGNAME=navasard HOME=/mloscratch/homes/navasard TORCHINDUCTOR_CACHE_DIR=/mloscratch/homes/navasard/.cache/torchinductor /mloscratch/homes/navasard/conda/bin/torchrun --standalone --nproc_per_node=4 train.py --config configs/p2_1p2b_mixed_seed72_4gpu_autoresume_current.json >> "$seed72_log" 2>&1 < /dev/null & | |