File size: 2,210 Bytes
7fde66e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | #!/bin/bash
# Usage: eval_checkpoint.sh <checkpoint-path> <tag> [n_tb2] [n_swe] [temp]
# Full loop: kill GPU procs, serve checkpoint, wait healthy, run both suites, shutdown.
set -u
CKPT=$1
TAG=$2
N_TB2=${3:-89}
N_SWE=${4:-100}
TEMP=${5:-}
PORT=8500
cd /root/work/a/prime-rl
export PRIME_API_KEY="$(cat "$AGENTPTB_PRIME_KEY_FILE")"
# kill anything holding GPUs
pkill -f "PRIME-RL" 2>/dev/null || true
sleep 5
pkill -9 -f "PRIME-RL" 2>/dev/null || true
sleep 3
# serve
CUDA_VISIBLE_DEVICES=0 nohup uv run inference \
--model.name "$CKPT" \
--model.tool-call-parser qwen3_coder \
--model.reasoning-parser qwen3 \
--server.port $PORT --parallel.tp 1 --parallel.dp 1 \
--vllm-extra '{"limit_mm_per_prompt": {"image": 0, "video": 0}}' \
> $AGENTPTB_WORKSPACE/logs/inference-$TAG.log 2>&1 &
SERVER_PID=$!
# wait for health
for i in $(seq 1 120); do
if curl -sf http://localhost:$PORT/health > /dev/null 2>&1; then
echo "server healthy after ${i}0s"
break
fi
sleep 10
done
curl -sf http://localhost:$PORT/health > /dev/null || { echo "SERVER FAILED"; exit 1; }
SAMPLING=()
if [ -n "$TEMP" ]; then
SAMPLING=(--sampling.temperature "$TEMP")
fi
OUT=$AGENTPTB_WORKSPACE/evals/$TAG
mkdir -p "$OUT"
echo "=== terminal-bench-2: $N_TB2 tasks ==="
uv run eval terminal-bench-2-v1 -m "$CKPT" \
--client.base-url "http://127.0.0.1:$PORT/v1" \
--env.agent.harness.id pi --env.agent.runtime.type broker \
--env.agent.runtime.ready_timeout_seconds 1800 \
"${SAMPLING[@]}" \
-n "$N_TB2" -r 1 -s -c 24 --no-push --rich false \
-o "$OUT/tb2" 2>&1 | tail -15
echo "=== swe-bench-verified: $N_SWE tasks ==="
uv run eval swebench-verified-v1 -m "$CKPT" \
--client.base-url "http://127.0.0.1:$PORT/v1" \
--env.agent.harness.id pi --env.agent.runtime.type broker \
--env.agent.runtime.ready_timeout_seconds 1800 \
"${SAMPLING[@]}" \
-n "$N_SWE" -r 1 -s -c 64 --no-push --rich false \
-o "$OUT/swe" 2>&1 | tail -15
# shutdown server, free GPUs
kill $SERVER_PID 2>/dev/null || true
sleep 5
pkill -9 -f "PRIME-RL" 2>/dev/null || true
echo "=== scores ==="
uv run python $AGENTPTB_WORKSPACE/scripts/score.py "$OUT/tb2/traces.jsonl" "$OUT/swe/traces.jsonl"
echo "=== DONE $TAG ==="
|