#!/bin/bash
set -u
. /workspace/.env; cd /workspace/aditi
while pgrep -f "[b]uild_corpus" >/dev/null; do sleep 30; done
cat corpus/aditi_v9a.jsonl corpus/aditi_v9b.jsonl > corpus/aditi_v9_full.jsonl 2>/dev/null || cp corpus/aditi_v9a.jsonl corpus/aditi_v9_full.jsonl
wc -l corpus/aditi_v9_full.jsonl >> logs/chain.log
echo "$(date -Is) v9 ready -> waiting for GPUs" >> logs/chain.log
while pgrep -f "[s]core.py" >/dev/null; do sleep 30; done
echo "$(date -Is) launching CAMPAIGN: r64 text (GPU0) + r32 trimodal (GPU1)" >> logs/chain.log
# ARM 1: the winning rank on the rebalanced corpus, long run
CUDA_VISIBLE_DEVICES=0 setsid /venv/main/bin/python pack_and_train.py \
  --corpus corpus/aditi_v9_full.jsonl --out runs/v9_r64 \
  --seq 2048 --accum 8 --rank 64 --lr 1.5e-4 --warmup 40 \
  --eval-blocks 128 --eval-every 100 --save-every 100 </dev/null > logs/train_v9_r64.log 2>&1 &
sleep 30
# ARM 2: tri-modal on the full multimodal set, warm-started from the winning text adapter
CUDA_VISIBLE_DEVICES=1 setsid /venv/main/bin/python train_mm.py \
  --corpus corpus/aditi_mm.jsonl --out runs/v9_mm \
  --rank 64 --lr 1e-4 --mm-lr 3e-5 --accum 8 --steps 1200 --warmup 40 --save-every 100 \
  </dev/null > logs/train_v9_mm.log 2>&1 &
echo "$(date -Is) campaign launched" >> logs/chain.log