File size: 2,628 Bytes
9589849 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | #!/usr/bin/env bash
set -euo pipefail
workspace="/mnt/pvc/users/simon/agentptb/runs/sol-max-s1/workspace"
mode="${1:-}"
case "$mode" in
review)
custom_kind="custom"
custom_label="custom"
;;
noreview)
custom_kind="custom-noreview"
custom_label="custom-noreview"
;;
stock)
custom_kind="custom-stock"
custom_label="custom-stock"
;;
*)
echo "usage: $0 {review|noreview|stock} {0.2|0.6}" >&2
exit 2
;;
esac
temperature="${2:-}"
case "$temperature" in
0.2) temperature_suffix="" ;;
0.6) temperature_suffix="-t06" ;;
*)
echo "usage: $0 {review|noreview|stock} {0.2|0.6}" >&2
exit 2
;;
esac
custom_tb2="$workspace/configs/eval-final-${custom_kind}${temperature_suffix}-tb2.toml"
custom_swe="$workspace/configs/eval-final-${custom_kind}${temperature_suffix}-swe.toml"
model="$workspace/submission/model"
[[ -L "$model" || -d "$model" ]]
[[ -f "$model/STABLE" ]]
curl --fail --silent --show-error http://127.0.0.1:8200/health >/dev/null
stock_tb2="$workspace/configs/eval-final-stock${temperature_suffix}-tb2.toml"
stock_swe="$workspace/configs/eval-final-stock${temperature_suffix}-swe.toml"
for output in \
"$workspace/evals/final-stock-tb2" \
"$workspace/evals/final-${custom_label}-tb2" \
"$workspace/evals/final-stock-swe" \
"$workspace/evals/final-${custom_label}-swe"; do
if [[ -e "$output" ]]; then
echo "refusing to overwrite existing final run: $output" >&2
exit 1
fi
done
cd "$workspace"
initial_status=0
scripts/run_paired_evals.sh \
"$stock_tb2" "$custom_tb2" "$stock_swe" "$custom_swe" \
|| initial_status=$?
printf 'initial_eval_exit_status=%s\n' "$initial_status" \
>state/final-initial-eval-status.txt
# Resume only missing episodes and explicit sandbox/network failures against
# the same saved configs. Model, harness, context, and task failures remain
# first-attempt outcomes and therefore remain failures in the denominator.
scripts/repair_eval_infra.sh \
evals/final-stock-tb2 \
"evals/final-${custom_label}-tb2" \
evals/final-stock-swe \
"evals/final-${custom_label}-swe"
for label in stock-tb2 "${custom_label}-tb2" stock-swe "${custom_label}-swe"; do
python scripts/summarize_eval.py "evals/final-${label}" >"logs/summary-final-${label}.json"
done
python scripts/compare_evals.py \
evals/final-stock-tb2 "evals/final-${custom_label}-tb2" \
--label-a stock --label-b custom >logs/compare-final-tb2.json
python scripts/compare_evals.py \
evals/final-stock-swe "evals/final-${custom_label}-swe" \
--label-a stock --label-b custom >logs/compare-final-swe.json
touch state/final-eval-complete
|