#!/usr/bin/env bash
# Run FELM + HaluEval benchmarks and write a polished report.
set -euo pipefail
cd "$(dirname "$0")/.."

mkdir -p eval/results/felm eval/results/halueval eval/reports logs
LOG="logs/benchmark_$(date -u +%Y%m%dT%H%M%SZ).log"
echo "Logging to $LOG"

{
  echo "===== START $(date -u +%Y-%m-%dT%H:%M:%SZ) ====="
  PAUSE="${PAUSE_SECONDS:-12}"
  echo "FELM... (pause=${PAUSE}s between API groups; resumes from cache)"
  uv run python -m eval.run --dataset felm --subset eval/subsets/felm_100.json \
    --results-dir eval/results --summary-out eval/results/felm/summary.json \
    --pause-seconds "$PAUSE"
  echo "HaluEval..."
  uv run python -m eval.run --dataset halueval --subset eval/subsets/halueval_40.json \
    --results-dir eval/results --summary-out eval/results/halueval/summary.json \
    --pause-seconds "$PAUSE"
  echo "Report..."
  uv run python -m eval.report --results-dir eval/results --out-dir eval/reports
  echo "===== DONE $(date -u +%Y-%m-%dT%H:%M:%SZ) ====="
} 2>&1 | tee -a "$LOG"