Spaces:
Sleeping
Sleeping
| # Run FELM + HaluEval benchmarks and write a polished report. | |
| set -euo pipefail | |
| cd "$(dirname "$0")/.." | |
| mkdir -p eval/results/felm eval/results/halueval eval/reports logs | |
| LOG="logs/benchmark_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| echo "Logging to $LOG" | |
| { | |
| echo "===== START $(date -u +%Y-%m-%dT%H:%M:%SZ) =====" | |
| PAUSE="${PAUSE_SECONDS:-12}" | |
| echo "FELM... (pause=${PAUSE}s between API groups; resumes from cache)" | |
| uv run python -m eval.run --dataset felm --subset eval/subsets/felm_100.json \ | |
| --results-dir eval/results --summary-out eval/results/felm/summary.json \ | |
| --pause-seconds "$PAUSE" | |
| echo "HaluEval..." | |
| uv run python -m eval.run --dataset halueval --subset eval/subsets/halueval_40.json \ | |
| --results-dir eval/results --summary-out eval/results/halueval/summary.json \ | |
| --pause-seconds "$PAUSE" | |
| echo "Report..." | |
| uv run python -m eval.report --results-dir eval/results --out-dir eval/reports | |
| echo "===== DONE $(date -u +%Y-%m-%dT%H:%M:%SZ) =====" | |
| } 2>&1 | tee -a "$LOG" | |