#!/usr/bin/env bash
set -euo pipefail

workspace="/mnt/pvc/users/simon/agentptb/runs/sol-max-s1/workspace"
mode="${1:-}"
case "$mode" in
  review)
    custom_kind="custom"
    custom_label="custom"
    ;;
  noreview)
    custom_kind="custom-noreview"
    custom_label="custom-noreview"
    ;;
  stock)
    custom_kind="custom-stock"
    custom_label="custom-stock"
    ;;
  *)
    echo "usage: $0 {review|noreview|stock} {0.2|0.6}" >&2
    exit 2
    ;;
esac
temperature="${2:-}"
case "$temperature" in
  0.2) temperature_suffix="" ;;
  0.6) temperature_suffix="-t06" ;;
  *)
    echo "usage: $0 {review|noreview|stock} {0.2|0.6}" >&2
    exit 2
    ;;
esac
custom_tb2="$workspace/configs/eval-final-${custom_kind}${temperature_suffix}-tb2.toml"
custom_swe="$workspace/configs/eval-final-${custom_kind}${temperature_suffix}-swe.toml"

model="$workspace/submission/model"
[[ -L "$model" || -d "$model" ]]
[[ -f "$model/STABLE" ]]
curl --fail --silent --show-error http://127.0.0.1:8200/health >/dev/null

stock_tb2="$workspace/configs/eval-final-stock${temperature_suffix}-tb2.toml"
stock_swe="$workspace/configs/eval-final-stock${temperature_suffix}-swe.toml"
for output in \
  "$workspace/evals/final-stock-tb2" \
  "$workspace/evals/final-${custom_label}-tb2" \
  "$workspace/evals/final-stock-swe" \
  "$workspace/evals/final-${custom_label}-swe"; do
  if [[ -e "$output" ]]; then
    echo "refusing to overwrite existing final run: $output" >&2
    exit 1
  fi
done

cd "$workspace"
initial_status=0
scripts/run_paired_evals.sh \
  "$stock_tb2" "$custom_tb2" "$stock_swe" "$custom_swe" \
  || initial_status=$?
printf 'initial_eval_exit_status=%s\n' "$initial_status" \
  >state/final-initial-eval-status.txt

# Resume only missing episodes and explicit sandbox/network failures against
# the same saved configs. Model, harness, context, and task failures remain
# first-attempt outcomes and therefore remain failures in the denominator.
scripts/repair_eval_infra.sh \
  evals/final-stock-tb2 \
  "evals/final-${custom_label}-tb2" \
  evals/final-stock-swe \
  "evals/final-${custom_label}-swe"

for label in stock-tb2 "${custom_label}-tb2" stock-swe "${custom_label}-swe"; do
  python scripts/summarize_eval.py "evals/final-${label}" >"logs/summary-final-${label}.json"
done
python scripts/compare_evals.py \
  evals/final-stock-tb2 "evals/final-${custom_label}-tb2" \
  --label-a stock --label-b custom >logs/compare-final-tb2.json
python scripts/compare_evals.py \
  evals/final-stock-swe "evals/final-${custom_label}-swe" \
  --label-a stock --label-b custom >logs/compare-final-swe.json
touch state/final-eval-complete