#!/usr/bin/env bash
# Prompting / scoring experiments on the top baseline models, compared against their zero-shot English logit run.
#   logit_si     Sinhala instruction, 4 cyclic option orders
#   logit_5shot  fixed 5 examples from the dev pool (same for every question), original option order
#   generate     greedy generation + answer parsing, original option order
#   cloze        length-normalised option likelihood (options not shown)
# Usage: scripts/run_experiments.sh [model ...]
set -uo pipefail
cd "$(dirname "$0")/../.."
PY=.venv/bin/python
DATA="sinhalammlu_dev gmmlu_si_val"
if [ $# -gt 0 ]; then MODELS=("$@"); else MODELS=(google/gemma-4-E4B-it google/gemma-3n-E4B-it Qwen/Qwen3.5-4B); fi

declare -A EXP=(
  [logit_si]="--method logit --lang si --n-perms 4"
  [logit_5shot]="--method logit --fewshot-file sinhalammlu_dev_5shot --n-shot 5 --n-perms 1"
  [generate]="--method generate --n-perms 1 --max-new-tokens 16"
  [cloze]="--method cloze"
)
ORDER=(logit_si logit_5shot generate cloze)

mkdir -p results/logs
for m in "${MODELS[@]}"; do
  for e in "${ORDER[@]}"; do
    name="exp_$(basename "$m")_${e}"
    if [ -f "results/$name/metrics.json" ]; then echo "skip $name (done)"; continue; fi
    echo "=== $name ($(date +%H:%M))"
    $PY -m eval.run_eval --model "$m" --data $DATA ${EXP[$e]} --run-name "$name" > "results/logs/$name.log" 2>&1 \
      && grep -E "acc=|scoring" "results/logs/$name.log" \
      || { echo "FAILED $name (see results/logs/$name.log)"; tail -5 "results/logs/$name.log"; }
  done
done
runs=()
for m in "${MODELS[@]}"; do runs+=("results/base_$(basename "$m")_logit" "results/exp_$(basename "$m")_*"); done
$PY -m eval.summarize --runs "${runs[@]}" --out reports/experiments_prompting.md > /dev/null
echo "summary -> reports/experiments_prompting.md"