File size: 1,609 Bytes
b2ebc95 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | #!/bin/bash
# Run expert-iteration collection only while the sandbox pool has real headroom.
#
# The pool oscillates between free (100-360 CPU) and jammed (60-100 pods queued behind other
# tenants). Sampling through a jam turns every episode into a SandboxError and deepens the
# queue for everyone, so this starts a round when there is room and kills it when there isn't.
#
# Tracks its child by pidfile: `pgrep -f <config path>` also matches any shell that merely
# mentions the path, which made an earlier version believe a round was permanently running.
WS=/mnt/pvc/users/simon/agentptb/runs/a-opus-max/workspace
cd "$WS" || exit 1
PIDF=/tmp/ei_round.pid
i=${EI_START_ROUND:-0}
while true; do
free=$(curl -s -m 30 "$SANDBOX_BASE_URL/resources" -H "X-API-Key: $SANDBOX_API_KEY" | python3 -c "
import json,sys,collections
try:
d=json.load(sys.stdin); c=collections.Counter(p['phase'] for p in d['sandbox_pods'])
print(1 if (c.get('Pending',0) < 8 and d['available']['cpu'] > 60) else 0)
except Exception: print(0)" 2>/dev/null)
alive=0
if [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; then alive=1; fi
if [ "$free" = "1" ] && [ "$alive" -eq 0 ]; then
i=$((i+1))
echo "$(date -Is) pool free -> ei round $i"
setsid nohup bash scripts/run_eval.sh "$WS/cfg/ei_r2e.toml" "$WS/runs/ei_round$i" \
> "logs/ei_round$i.log" 2>&1 < /dev/null &
echo $! > "$PIDF"
elif [ "$free" = "0" ] && [ "$alive" -eq 1 ]; then
echo "$(date -Is) pool jammed -> pausing round $i"
kill -TERM "$(cat "$PIDF")" 2>/dev/null
sleep 20
rm -f "$PIDF"
fi
sleep 600
done
|