| #!/usr/bin/env bash |
| set -euo pipefail |
|
|
| workspace="/mnt/pvc/users/simon/agentptb/runs/sol-max-s1/workspace" |
| mode="${1:-}" |
| case "$mode" in |
| review) |
| custom_kind="custom" |
| custom_label="custom" |
| ;; |
| noreview) |
| custom_kind="custom-noreview" |
| custom_label="custom-noreview" |
| ;; |
| stock) |
| custom_kind="custom-stock" |
| custom_label="custom-stock" |
| ;; |
| *) |
| echo "usage: $0 {review|noreview|stock} {0.2|0.6}" >&2 |
| exit 2 |
| ;; |
| esac |
| temperature="${2:-}" |
| case "$temperature" in |
| 0.2) temperature_suffix="" ;; |
| 0.6) temperature_suffix="-t06" ;; |
| *) |
| echo "usage: $0 {review|noreview|stock} {0.2|0.6}" >&2 |
| exit 2 |
| ;; |
| esac |
| custom_tb2="$workspace/configs/eval-final-${custom_kind}${temperature_suffix}-tb2.toml" |
| custom_swe="$workspace/configs/eval-final-${custom_kind}${temperature_suffix}-swe.toml" |
|
|
| model="$workspace/submission/model" |
| [[ -L "$model" || -d "$model" ]] |
| [[ -f "$model/STABLE" ]] |
| curl --fail --silent --show-error http://127.0.0.1:8200/health >/dev/null |
|
|
| stock_tb2="$workspace/configs/eval-final-stock${temperature_suffix}-tb2.toml" |
| stock_swe="$workspace/configs/eval-final-stock${temperature_suffix}-swe.toml" |
| for output in \ |
| "$workspace/evals/final-stock-tb2" \ |
| "$workspace/evals/final-${custom_label}-tb2" \ |
| "$workspace/evals/final-stock-swe" \ |
| "$workspace/evals/final-${custom_label}-swe"; do |
| if [[ -e "$output" ]]; then |
| echo "refusing to overwrite existing final run: $output" >&2 |
| exit 1 |
| fi |
| done |
|
|
| cd "$workspace" |
| initial_status=0 |
| scripts/run_paired_evals.sh \ |
| "$stock_tb2" "$custom_tb2" "$stock_swe" "$custom_swe" \ |
| || initial_status=$? |
| printf 'initial_eval_exit_status=%s\n' "$initial_status" \ |
| >state/final-initial-eval-status.txt |
|
|
| |
| |
| |
| scripts/repair_eval_infra.sh \ |
| evals/final-stock-tb2 \ |
| "evals/final-${custom_label}-tb2" \ |
| evals/final-stock-swe \ |
| "evals/final-${custom_label}-swe" |
|
|
| for label in stock-tb2 "${custom_label}-tb2" stock-swe "${custom_label}-swe"; do |
| python scripts/summarize_eval.py "evals/final-${label}" >"logs/summary-final-${label}.json" |
| done |
| python scripts/compare_evals.py \ |
| evals/final-stock-tb2 "evals/final-${custom_label}-tb2" \ |
| --label-a stock --label-b custom >logs/compare-final-tb2.json |
| python scripts/compare_evals.py \ |
| evals/final-stock-swe "evals/final-${custom_label}-swe" \ |
| --label-a stock --label-b custom >logs/compare-final-swe.json |
| touch state/final-eval-complete |
|
|
