| # 19 Oct, step 1: does thinking mode help? Model A (current submission model) and the base model, think-then-answer, | |
| # on Global-MMLU academic subjects (STEM + Business, proxy for A/L "hard") and a 400-question dev sample. | |
| set -uo pipefail | |
| cd "$(dirname "$0")/../.." | |
| PY=.venv/bin/python | |
| A=models/sft_gmmlu_lora/step-357 | |
| run() { # name model data... | |
| local name=$1 m=$2; shift 2 | |
| [ -f "results/$name/metrics.json" ] && { echo "skip $name"; return; } | |
| $PY -m eval.run_eval --model "$m" --data "$@" --method generate --thinking --max-new-tokens 2048 \ | |
| --run-name "$name" > "results/logs/$name.log" 2>&1 && grep -E "acc=" "results/logs/$name.log" | sed 's/.*\(think_\)/\1/' \ | |
| || { echo "FAILED $name"; tail -3 "results/logs/$name.log"; } | |
| } | |
| echo "=== thinking tests ($(date '+%H:%M'))" | |
| run think_A_academic "$A" gmmlu_si_val_academic | |
| run think_A_dev400 "$A" smmlu_dev_sample400 | |
| run think_base_academic models/gemma-4-E4B-it-text gmmlu_si_val_academic | |
| echo "=== done ($(date '+%H:%M'))" | |