| set -u | |
| . /workspace/.env; cd /workspace/aditi | |
| while pgrep -f "[b]uild_corpus" >/dev/null; do sleep 30; done | |
| cat corpus/aditi_v9a.jsonl corpus/aditi_v9b.jsonl > corpus/aditi_v9_full.jsonl 2>/dev/null || cp corpus/aditi_v9a.jsonl corpus/aditi_v9_full.jsonl | |
| wc -l corpus/aditi_v9_full.jsonl >> logs/chain.log | |
| echo "$(date -Is) v9 ready -> waiting for GPUs" >> logs/chain.log | |
| while pgrep -f "[s]core.py" >/dev/null; do sleep 30; done | |
| echo "$(date -Is) launching CAMPAIGN: r64 text (GPU0) + r32 trimodal (GPU1)" >> logs/chain.log | |
| # ARM 1: the winning rank on the rebalanced corpus, long run | |
| CUDA_VISIBLE_DEVICES=0 setsid /venv/main/bin/python pack_and_train.py \ | |
| --corpus corpus/aditi_v9_full.jsonl --out runs/v9_r64 \ | |
| --seq 2048 --accum 8 --rank 64 --lr 1.5e-4 --warmup 40 \ | |
| --eval-blocks 128 --eval-every 100 --save-every 100 </dev/null > logs/train_v9_r64.log 2>&1 & | |
| sleep 30 | |
| # ARM 2: tri-modal on the full multimodal set, warm-started from the winning text adapter | |
| CUDA_VISIBLE_DEVICES=1 setsid /venv/main/bin/python train_mm.py \ | |
| --corpus corpus/aditi_mm.jsonl --out runs/v9_mm \ | |
| --rank 64 --lr 1e-4 --mm-lr 3e-5 --accum 8 --steps 1200 --warmup 40 --save-every 100 \ | |
| </dev/null > logs/train_v9_mm.log 2>&1 & | |
| echo "$(date -Is) campaign launched" >> logs/chain.log | |