| #!/usr/bin/env bash |
| set -eo pipefail |
|
|
| SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| |
| source "${SCRIPT_DIR}/conda_env.sh" |
| set -u |
|
|
|
|
| ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" |
| cd "${ROOT}" |
|
|
| BASE_CONFIG="${BASE_CONFIG:-configs/grpo_llama32_3b_bf16.yaml}" |
| ACCELERATE_CONFIG="${ACCELERATE_CONFIG:-configs/accelerate_ddp_4gpu.yaml}" |
| NUM_PROCESSES="${NUM_PROCESSES:-4}" |
|
|
| MODEL_DIR="${MODEL_DIR:-}" |
| if [[ -z "${MODEL_DIR}" ]]; then |
| echo "Missing MODEL_DIR. Example:" |
| echo " MODEL_DIR=/path/to/checkpoint OUTPUT_PATH=out.json ./scripts/eval_math_level1_4gpu.sh" |
| exit 1 |
| fi |
|
|
| OUTPUT_PATH="${OUTPUT_PATH:-artifacts/eval/math_level1/accuracy.json}" |
| MAX_SAMPLES="${MAX_SAMPLES:--1}" |
| BATCH_SIZE="${BATCH_SIZE:-4}" |
| MATH_SPLIT="${MATH_SPLIT:-test}" |
| SAVE_ROLLOUTS="${SAVE_ROLLOUTS:-0}" |
| ROLLOUTS_MAX_SAMPLES="${ROLLOUTS_MAX_SAMPLES:-50}" |
| ROLLOUTS_OUT_PATH="${ROLLOUTS_OUT_PATH:-artifacts/eval/math_level1/rollouts.jsonl}" |
| MAX_PROMPT_LENGTH="${MAX_PROMPT_LENGTH:-}" |
| MAX_COMPLETION_LENGTH="${MAX_COMPLETION_LENGTH:-}" |
|
|
| export PYTHONPATH="${ROOT}/src" |
| export BASE_CONFIG |
| export MODEL_DIR |
| export OUTPUT_PATH |
| export MAX_SAMPLES |
| export BATCH_SIZE |
| export MATH_SPLIT |
| export SAVE_ROLLOUTS |
| export ROLLOUTS_MAX_SAMPLES |
| export ROLLOUTS_OUT_PATH |
| if [[ -n "${MAX_PROMPT_LENGTH}" ]]; then export MAX_PROMPT_LENGTH; fi |
| if [[ -n "${MAX_COMPLETION_LENGTH}" ]]; then export MAX_COMPLETION_LENGTH; fi |
|
|
| export WANDB_MODE="${WANDB_MODE:-offline}" |
| export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}" |
| export HF_DATASETS_OFFLINE="${HF_DATASETS_OFFLINE:-1}" |
| export TRANSFORMERS_OFFLINE="${TRANSFORMERS_OFFLINE:-1}" |
|
|
| accelerate launch \ |
| --config_file "${ACCELERATE_CONFIG}" \ |
| --num_processes "${NUM_PROCESSES}" \ |
| src/eval_math_level1.py |
|
|
