| set -u | |
| . /workspace/.env; cd /workspace/aditi | |
| # wait for both text arms | |
| while pgrep -f "[p]ack_and_train.py" >/dev/null; do sleep 60; done | |
| echo "$(date -Is) text arms done -> tri-modal smoke test" >> logs/chain.log | |
| # 3-step smoke test first: catch collator bugs before committing hours | |
| CUDA_VISIBLE_DEVICES=0 /venv/main/bin/python train_mm.py \ | |
| --corpus corpus/aditi_mm.jsonl --out runs/mm_smoke \ | |
| --rank 32 --accum 2 --steps 3 --warmup 1 --save-every 999 > logs/mm_smoke.log 2>&1 | |
| if grep -q "\[step 3/3\]" logs/mm_smoke.log; then | |
| echo "$(date -Is) smoke PASSED -> full tri-modal run" >> logs/chain.log | |
| # full tri-modal: text corpus + multimodal, warm-started from the best text adapter | |
| CUDA_VISIBLE_DEVICES=0 setsid /venv/main/bin/python train_mm.py \ | |
| --corpus corpus/aditi_mm.jsonl --out runs/mm_r32 \ | |
| --rank 32 --lr 1e-4 --mm-lr 2e-5 --accum 8 --steps 480 --warmup 20 --save-every 60 \ | |
| </dev/null > logs/train_mm_r32.log 2>&1 & | |
| else | |
| echo "$(date -Is) smoke FAILED - see logs/mm_smoke.log" >> logs/chain.log | |
| fi | |