#!/usr/bin/env bash
# Self-pacing, resumable, quota-aware full 10-task gate driver.
# Waits for flash-lite quota (+tunnel), runs ONE task at a time (1 GPU),
# keeps clean results, wipes+retries quota-hit tasks (<=4 attempts), grinds across days.
LOG=/root/gate_flash_main.log
OUT=/root/autodl-tmp/graph-as-policy/outputs/gate_flash
log(){ echo "[$(date "+%F %T")] $*" >> "$LOG"; }
probe(){ K=$(cat /root/.gemini_key); c=$(curl -s -o /dev/null -w "%{http_code}" --max-time 20 -x http://127.0.0.1:18899 "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions" -H "Authorization: Bearer $K" -H "Content-Type: application/json" -d "{\"model\":\"gemini-3.1-flash-lite-preview\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}],\"max_tokens\":3}"); [ "$c" = "200" ]; }
cd /root/autodl-tmp/graph-as-policy || exit 1
mkdir -p "$OUT"
log "==== SELF-PACING GATE DRIVER START (waits for daily quota reset) ===="
for t in 0 1 2 3 4 5 6 7 8 9; do
  [ -f "$OUT/t$t.done" ] && { log "task $t already clean -> skip"; continue; }
  att=0
  while [ $att -lt 4 ]; do
    att=$((att+1))
    while ! probe; do log "task $t att$att: quota/tunnel down, sleep 30m"; sleep 1800; done
    log "task $t att$att: quota OK -> running"
    source /root/repro_gemini_env.sh
    rm -rf "$OUT/t$t"
    uv run gap benchmark examples/benchmark/solo_flash_t${t}.yaml --output-dir "$OUT/t$t" > /root/gate_flash_t${t}.log 2>&1
    a=$(find "$OUT/t$t" -type d -name "trial_*reward*" 2>/dev/null | wc -l)
    p=$(find "$OUT/t$t" -type d -name "trial_*reward1*pass" 2>/dev/null | wc -l)
    q=$(grep -c "backend unavailable" /root/gate_flash_t${t}.log 2>/dev/null)
    if [ "$a" -ge 3 ] && [ "$q" -eq 0 ]; then
      touch "$OUT/t$t.done"; log "task $t CLEAN: pass=$p/$a (att$att)"; break
    fi
    log "task $t att$att incomplete (trials=$a quota_aborts=$q) -> retry"; sleep 300
  done
  [ -f "$OUT/t$t.done" ] || log "task $t GAVE UP after 4 attempts"
  sleep 120
done
TP=0; TA=0
for t in 0 1 2 3 4 5 6 7 8 9; do
  a=$(find "$OUT/t$t" -type d -name "trial_*reward*" 2>/dev/null | wc -l)
  p=$(find "$OUT/t$t" -type d -name "trial_*reward1*pass" 2>/dev/null | wc -l)
  log "  task $t: $p/$a"; TP=$((TP+p)); TA=$((TA+a))
done
log "==== AGGREGATE: $TP/$TA passed  ===="