#!/bin/bash
# Run expert-iteration collection only while the sandbox pool has real headroom.
#
# The pool oscillates between free (100-360 CPU) and jammed (60-100 pods queued behind other
# tenants). Sampling through a jam turns every episode into a SandboxError and deepens the
# queue for everyone, so this starts a round when there is room and kills it when there isn't.
#
# Tracks its child by pidfile: `pgrep -f <config path>` also matches any shell that merely
# mentions the path, which made an earlier version believe a round was permanently running.
WS=/mnt/pvc/users/simon/agentptb/runs/a-opus-max/workspace
cd "$WS" || exit 1
PIDF=/tmp/ei_round.pid
i=${EI_START_ROUND:-0}
while true; do
  free=$(curl -s -m 30 "$SANDBOX_BASE_URL/resources" -H "X-API-Key: $SANDBOX_API_KEY" | python3 -c "
import json,sys,collections
try:
    d=json.load(sys.stdin); c=collections.Counter(p['phase'] for p in d['sandbox_pods'])
    print(1 if (c.get('Pending',0) < 8 and d['available']['cpu'] > 60) else 0)
except Exception: print(0)" 2>/dev/null)
  alive=0
  if [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; then alive=1; fi
  if [ "$free" = "1" ] && [ "$alive" -eq 0 ]; then
    i=$((i+1))
    echo "$(date -Is) pool free -> ei round $i"
    setsid nohup bash scripts/run_eval.sh "$WS/cfg/ei_r2e.toml" "$WS/runs/ei_round$i" \
        > "logs/ei_round$i.log" 2>&1 < /dev/null &
    echo $! > "$PIDF"
  elif [ "$free" = "0" ] && [ "$alive" -eq 1 ]; then
    echo "$(date -Is) pool jammed -> pausing round $i"
    kill -TERM "$(cat "$PIDF")" 2>/dev/null
    sleep 20
    rm -f "$PIDF"
  fi
  sleep 600
done