File size: 3,477 Bytes
439e7b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | #!/usr/bin/env bash
# Fine-tuning experiment queue (the instructor's dials: loss function, epochs, layers, plus the
# pretraining and synthetic-data ablations). Waits for the pretraining run to finish, then runs
# each experiment sequentially on the GPU, evaluates its best/ on the TREC-06 binary holdout and
# the three-way holdout, and exports the main run. Launch in tmux:
#
# tmux new-session -d -s phish-finetune 'scripts/run_experiments.sh 2>&1 | tee logs/experiments_$(date +%Y%m%d-%H%M%S).out'
#
# Resumable: a run whose best/ already exists is skipped; re-run the script to continue after a crash.
set -uo pipefail
cd "$(dirname "$0")/.."
PY=".venv/bin/python"
PRETRAIN="checkpoints/pretrain/${PRETRAIN_RUN:-large-24L}/final"
EVAL_EVERY="${EVAL_EVERY:-1250}" # ~quarter epoch at batch 32 over 160k rows (val eval costs ~1 min)
MAIN="${MAIN_RUN:-ft-ce}"
log() { printf '%s %s\n' "$(date +%H:%M:%S)" "$*"; }
if [[ ! -f "$PRETRAIN/model.safetensors" ]]; then
log "waiting for $PRETRAIN/model.safetensors (pretraining still running)"
while [[ ! -f "$PRETRAIN/model.safetensors" ]]; do sleep 300; done
log "pretraining final/ found; waiting 60 s for the GPU to free"; sleep 60
fi
$PY src/pretrain.py plot --run-name "${PRETRAIN_RUN:-large-24L}" || true
# name | extra flags (all: --init pretrained:$PRETRAIN unless --init scratch is given)
EXPERIMENTS=(
"ft-ce --loss cross_entropy --epochs 3"
"ft-weighted --loss weighted_cross_entropy --epochs 2"
"ft-focal --loss focal --epochs 2"
"ft-synthetic --loss cross_entropy --epochs 2 --synthetic on"
"ft-12L --loss cross_entropy --epochs 2 --n-layers 12"
"ft-scratch --loss cross_entropy --epochs 2 --init scratch"
)
STATUS=()
for spec in "${EXPERIMENTS[@]}"; do
name="${spec%% *}"; flags="${spec#* }"
if [[ -f "checkpoints/finetune/$name/best/model.safetensors" ]]; then
log "$name: best/ exists, skipping training"
else
init="--init pretrained:$PRETRAIN"; [[ "$flags" == *"--init scratch"* ]] && init=""
log "$name: $PY src/finetune.py train --run-name $name $init $flags --eval-every $EVAL_EVERY"
$PY src/finetune.py train --run-name "$name" $init $flags --eval-every "$EVAL_EVERY"
rc=$?; log "$name: finetune exit $rc"; STATUS+=("$name:train=$rc")
[[ $rc -ne 0 ]] && continue
fi
$PY src/finetune.py plot --run-name "$name" --out "reports/finetune_curves_$name.png" || true
if [[ ! -f "reports/eval_done_$name" ]]; then
# one invocation = one experiments.jsonl line + one RUN_LOG entry covering both holdouts
$PY src/evaluate.py run --checkpoint "checkpoints/finetune/$name/best" --split test,test3 --run-name "$name" --out-dir "reports/eval_$name"
rc=$?; log "$name: evaluate exit $rc"; STATUS+=("$name:eval=$rc")
[[ $rc -eq 0 ]] && touch "reports/eval_done_$name"
fi
done
# main run: top-level reports + export + app files
if [[ -f "checkpoints/finetune/$MAIN/best/model.safetensors" ]]; then
$PY src/finetune.py plot --run-name "$MAIN" || true
$PY src/evaluate.py run --checkpoint "checkpoints/finetune/$MAIN/best" --split test,test3 --run-name "$MAIN" \
--finetune-run-dir "checkpoints/finetune/$MAIN" --pretrain-run-dir "checkpoints/pretrain/${PRETRAIN_RUN:-large-24L}"
$PY src/export.py run --checkpoint "checkpoints/finetune/$MAIN/best" --out-dir "model_export/$MAIN" --app-dir app
STATUS+=("$MAIN:export=$?")
fi
$PY src/finetune.py list
log "queue finished: ${STATUS[*]:-nothing ran}"
|