eval/run_eval.sh

#!/bin/bash
# ================================================================
# DM-GRPO Final Evaluation Launcher
# ================================================================
# Usage:
#   bash run_eval.sh                          # Run all models in the list
#   bash run_eval.sh --gpus 0,1,2,3           # Specify GPUs
#   bash run_eval.sh --benchmarks tempcompass  # Specify benchmarks
#   tmux new-session -d -s final_eval "bash /root/zwj/DM-GRPO/Eval/run_eval.sh --gpus 0,1,2,3"
# To add a new model: append to MODELS array below.
# Format: "model_path|file_name|nframes|rep_penalty"
# ================================================================

set -e
source /root/miniconda3/etc/profile.d/conda.sh
conda activate Video-R1

export DECORD_EOF_RETRY_MAX=20480

# ========================== Configuration ==========================
# Format: "model_path|file_name|nframes|rep_penalty"
# nframes: 16 or 32 (set to match training)
# rep_penalty: 1.0 (greedy) or 1.1 (with penalty)
MODELS=(
    # --- Add models here ---
    # "/root/zwj/Model/Qwen2.5-VL-7B-Instruct|Qwen2.5-VL-7B-Instruct-32f-1.0|32|1.0"
    # "/root/zwj/Model/Qwen2.5-VL-7B-Instruct|Qwen2.5-VL-7B-Instruct-32f-1.1|32|1.1"
    # "/root/zwj/Video-R1/src/r1-v/log/DeltaMask-GRPO-v5_SFT-v9-16f_ts-reward|DM-GRPO-v5_SFT-v9-16f_rep1.1|32|1.1"
    # "/root/zwj/Video-R1/src/r1-v/log/DeltaMask-GRPO-v4_SFT-v9-32f_ts-reward|DM-GRPO-v4_SFT-v9-32f_rep1.1|32|1.1"
    "/root/zwj/Model/SFT/Qwen2.5-VL-7B-Video-CoT-SFT-v5-same-speed-32f|SFT-v5-same-speed-32f_rep1.0|32|1.0"
    # "/root/zwj/Model/SFT/Qwen2.5-VL-7B-Video-CoT-SFT-v5-same-speed-32f|SFT-v5-same-speed-32f_rep1.1|32|1.1"
)

# Defaults (override via CLI args)
GPUS="${GPUS:-0,1,2,3}"
BENCHMARKS="${BENCHMARKS:-tempcompass}"
BSZ="${BSZ:-64}"
MAX_TOKENS="${MAX_TOKENS:-4096}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
GPU_MEM="${GPU_MEM:-0.8}"

# ========================== Parse CLI args ==========================
while [[ $# -gt 0 ]]; do
    case $1 in
        --gpus) GPUS="$2"; shift 2;;
        --benchmarks) BENCHMARKS="$2"; shift 2;;
        --bsz) BSZ="$2"; shift 2;;
        --max_tokens) MAX_TOKENS="$2"; shift 2;;
        *) echo "Unknown arg: $1"; exit 1;;
    esac
done

export CUDA_VISIBLE_DEVICES="$GPUS"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# ========================== Run ==========================
if [ ${#MODELS[@]} -eq 0 ]; then
    echo "ERROR: No models configured. Edit MODELS array in this script."
    exit 1
fi

for entry in "${MODELS[@]}"; do
    IFS='|' read -r MODEL_PATH FILE_NAME NFRAMES REP_PENALTY <<< "$entry"

    echo ""
    echo "============================================================"
    echo "  Model:        $MODEL_PATH"
    echo "  File name:    $FILE_NAME"
    echo "  Frames:       $NFRAMES"
    echo "  rep_penalty:  $REP_PENALTY"
    echo "  Benchmarks:   $BENCHMARKS"
    echo "  GPUs:         $GPUS"
    echo "============================================================"

    NFRAMES_ARG=""
    if [ -n "$NFRAMES" ] && [ "$NFRAMES" != "none" ]; then
        NFRAMES_ARG="--nframes $NFRAMES"
    fi

    python "$SCRIPT_DIR/eval.py" \
        --model_path "$MODEL_PATH" \
        --file_name "$FILE_NAME" \
        --benchmarks $BENCHMARKS \
        --bsz "$BSZ" \
        --max_tokens "$MAX_TOKENS" \
        --max_model_len "$MAX_MODEL_LEN" \
        --gpu_memory_utilization "$GPU_MEM" \
        --repetition_penalty "$REP_PENALTY" \
        $NFRAMES_ARG

    echo "Done: $FILE_NAME"
done

echo ""
echo "All evaluations complete."