#!/bin/bash
set -u
. /workspace/.env; cd /workspace/aditi
# wait for both text arms
while pgrep -f "[p]ack_and_train.py" >/dev/null; do sleep 60; done
echo "$(date -Is) text arms done -> tri-modal smoke test" >> logs/chain.log
# 3-step smoke test first: catch collator bugs before committing hours
CUDA_VISIBLE_DEVICES=0 /venv/main/bin/python train_mm.py \
  --corpus corpus/aditi_mm.jsonl --out runs/mm_smoke \
  --rank 32 --accum 2 --steps 3 --warmup 1 --save-every 999 > logs/mm_smoke.log 2>&1
if grep -q "\[step 3/3\]" logs/mm_smoke.log; then
  echo "$(date -Is) smoke PASSED -> full tri-modal run" >> logs/chain.log
  # full tri-modal: text corpus + multimodal, warm-started from the best text adapter
  CUDA_VISIBLE_DEVICES=0 setsid /venv/main/bin/python train_mm.py \
    --corpus corpus/aditi_mm.jsonl --out runs/mm_r32 \
    --rank 32 --lr 1e-4 --mm-lr 2e-5 --accum 8 --steps 480 --warmup 20 --save-every 60 \
    </dev/null > logs/train_mm_r32.log 2>&1 &
else
  echo "$(date -Is) smoke FAILED - see logs/mm_smoke.log" >> logs/chain.log
fi