Upload folder using huggingface_hub

2e7f2ce verified 7 months ago

7.12 kB

#!/bin/bash
# =============================================================
# VoRA Evaluation Runner (打包版)
# =============================================================
#
# ========== 使用教程 ==========
#
# 1. 把 eval_pack/ 整个文件夹传到目标机器
#
# 2. 创建 conda 环境 (一次性):
#      conda create -n eval python=3.10 -y
#      conda activate eval
#      cd eval_pack
#      pip install -r requirements_eval.txt
#
# 3. 修改下方「路径配置」部分:
#      - QWEN_MODEL:      Qwen2.5-7B-Instruct 路径
#      - QWEN_VL_MODEL:   Qwen2.5-VL-3B-Instruct 路径
#      - AIMV2_PATH:      aimv2-huge-patch14-448 路径
#      - VORA_CIRC_MERGED: VoRA-Circulant merged 模型路径
#
# 4. 修改 data/eval_qwenvl.jsonl 中的图片绝对路径:
#      sed -i 's|/share/home/jcdl1lsy2clx/VoRA/data_dir/VoRA-Recap-29M/frames|/你的/frames路径|g' data/eval_qwenvl.jsonl
#
#    data/eval_vora.jsonl 用的是相对路径 (frames/xxx.jpg),
#    确保从 eval_pack/ 目录能访问到 frames/ 文件夹:
#      ln -s /你的/frames路径 frames
#
# 5. 运行:
#      conda activate eval
#      cd eval_pack
#      bash eval/run_eval.sh
#
# 6. 结果会打印表格并保存到 eval/results/*.json
#
# ========== 目录结构 ==========
#
#   eval_pack/
#   ├── eval/                <- 评测脚本
#   ├── models/              <- VoRA 模型代码
#   ├── tools/               <- merge_lora.py
#   ├── generation_files/    <- tokenizer/processor
#   ├── data/
#   │   ├── eval_vora.jsonl  <- VoRA 系列用 (相对路径)
#   │   └── eval_qwenvl.jsonl <- Qwen-VL 系列用 (绝对路径)
#   ├── vora_merged_250/     <- VoRA merged (~32G)
#   ├── lora_merged/         <- Qwen-VL+LoRA merged (~7G)
#   ├── circulant_merged/    <- Qwen-VL+Circulant merged (~7G)
#   └── requirements_eval.txt
#
# =============================================================

set -e
cd "$(dirname "$0")/.."

# ---------- 路径配置 (根据你的环境修改) ----------
# 基线模型 (另一台机器上已有)
QWEN_MODEL="/path/to/Qwen2.5-7B-Instruct"
QWEN_VL_MODEL="/path/to/Qwen2.5-VL-3B-Instruct"
AIMV2_PATH="/path/to/aimv2-huge-patch14-448"

# 评测数据
EVAL_DATA_VORA="data/eval_vora.jsonl"       # VoRA 系列用 (相对路径 frames/xxx.jpg)
EVAL_DATA_QWEN="data/eval_qwenvl.jsonl"     # Qwen-VL 系列用 (绝对路径, 需要按目标机器修改)

DTYPE="float16"
RESULTS_DIR="eval/results"

# 已 merge 好的模型
VORA_MERGED="vora_merged_250"
VORA_CIRC_MERGED="/path/to/vora-circulant-merged"
LORA_MERGED="lora_merged"
CIRC_MERGED="circulant_merged"
# -----------------------------------------------

mkdir -p "$RESULTS_DIR"

pip install nltk rouge-score qwen-vl-utils 2>/dev/null || true
python -c "import nltk; nltk.download('punkt', quiet=True); nltk.download('punkt_tab', quiet=True)" 2>/dev/null || true

echo ""
echo "=============================================="
echo "  Step 1: Qwen2.5-7B Text-Only Baseline"
echo "=============================================="
python eval/eval_qwen_baseline.py \
    --mode all \
    --model-path "$QWEN_MODEL" \
    --eval-data "$EVAL_DATA_QWEN" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/qwen_baseline.json"

echo ""
echo "=============================================="
echo "  Step 2: VoRA Full Eval"
echo "=============================================="
python eval/eval_vora.py \
    --mode all \
    --checkpoint "$VORA_MERGED" \
    --merged \
    --eval-data "$EVAL_DATA_VORA" \
    --image-processor "$AIMV2_PATH" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/vora_best.json"

echo ""
echo "=============================================="
echo "  Step 3: VoRA-Circulant Full Eval"
echo "=============================================="
python eval/eval_vora.py \
    --mode all \
    --checkpoint "$VORA_CIRC_MERGED" \
    --merged \
    --eval-data "$EVAL_DATA_VORA" \
    --image-processor "$AIMV2_PATH" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/vora_circulant.json"

echo ""
echo "=============================================="
echo "  Step 4: Qwen2.5-VL-3B Original Baseline"
echo "=============================================="
python eval/eval_qwen_vl.py \
    --mode all \
    --model-path "$QWEN_VL_MODEL" \
    --eval-data "$EVAL_DATA_QWEN" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/qwen_vl_original.json"

echo ""
echo "=============================================="
echo "  Step 5: Qwen2.5-VL-3B + LoRA"
echo "=============================================="
if [ -d "$LORA_MERGED" ]; then
    python eval/eval_qwen_vl.py \
        --mode all \
        --model-path "$LORA_MERGED" \
        --eval-data "$EVAL_DATA_QWEN" \
        --max-samples 200 \
        --max-new-tokens 256 \
        --dtype "$DTYPE" \
        --output "$RESULTS_DIR/qwen_vl_lora.json"
else
    echo "LoRA merged model not found at $LORA_MERGED, skipping."
fi

echo ""
echo "=============================================="
echo "  Step 6: Qwen2.5-VL-3B + Block-Circulant"
echo "=============================================="
if [ -d "$CIRC_MERGED" ]; then
    python eval/eval_qwen_vl.py \
        --mode all \
        --model-path "$CIRC_MERGED" \
        --eval-data "$EVAL_DATA_QWEN" \
        --max-samples 200 \
        --max-new-tokens 256 \
        --dtype "$DTYPE" \
        --output "$RESULTS_DIR/qwen_vl_circulant.json"
else
    echo "Block-Circulant merged model not found at $CIRC_MERGED, skipping."
fi

echo ""
echo "=============================================="
echo "  Summary"
echo "=============================================="
python -c "
import json, os, math
results_dir = '$RESULTS_DIR'
print('='*70)
print(f'{\"Model\":<35} {\"Loss\":>10} {\"PPL\":>8} {\"BLEU-4\":>8} {\"ROUGE-L\":>8}')
print('-'*70)
def print_row(name, filepath):
    if not os.path.exists(filepath):
        return
    r = json.load(open(filepath))
    ppl = r.get('perplexity', None)
    loss = math.log(ppl) if ppl and ppl != float('inf') else None
    b4 = r.get('BLEU-4', None)
    rl = r.get('ROUGE-L', None)
    loss_s = f'{loss:.4f}' if loss else '-'
    ppl_s = f'{ppl:.2f}' if ppl else '-'
    b4_s = f'{b4:.4f}' if b4 else '-'
    rl_s = f'{rl:.4f}' if rl else '-'
    print(f'{name:<35} {loss_s:>10} {ppl_s:>8} {b4_s:>8} {rl_s:>8}')
print_row('Qwen2.5-7B (text-only)', os.path.join(results_dir, 'qwen_baseline.json'))
print_row('VoRA', os.path.join(results_dir, 'vora_best.json'))
print_row('VoRA-Circulant', os.path.join(results_dir, 'vora_circulant.json'))
print('-'*70)
print_row('Qwen2.5-VL-3B (original)', os.path.join(results_dir, 'qwen_vl_original.json'))
print_row('Qwen2.5-VL-3B + LoRA', os.path.join(results_dir, 'qwen_vl_lora.json'))
print_row('Qwen2.5-VL-3B + Circulant', os.path.join(results_dir, 'qwen_vl_circulant.json'))
print('='*70)
"

echo ""
echo "All results saved to \$RESULTS_DIR/"
echo "Done!"

	#!/bin/bash
	# =============================================================
	# VoRA Evaluation Runner (打包版)
	# =============================================================
	#
	# ========== 使用教程 ==========
	#
	# 1. 把 eval_pack/ 整个文件夹传到目标机器
	#
	# 2. 创建 conda 环境 (一次性):
	# conda create -n eval python=3.10 -y
	# conda activate eval
	# cd eval_pack
	# pip install -r requirements_eval.txt
	#
	# 3. 修改下方「路径配置」部分:
	# - QWEN_MODEL: Qwen2.5-7B-Instruct 路径
	# - QWEN_VL_MODEL: Qwen2.5-VL-3B-Instruct 路径
	# - AIMV2_PATH: aimv2-huge-patch14-448 路径
	# - VORA_CIRC_MERGED: VoRA-Circulant merged 模型路径
	#
	# 4. 修改 data/eval_qwenvl.jsonl 中的图片绝对路径:
	# sed -i 's\|/share/home/jcdl1lsy2clx/VoRA/data_dir/VoRA-Recap-29M/frames\|/你的/frames路径\|g' data/eval_qwenvl.jsonl
	#
	# data/eval_vora.jsonl 用的是相对路径 (frames/xxx.jpg),
	# 确保从 eval_pack/ 目录能访问到 frames/ 文件夹:
	# ln -s /你的/frames路径 frames
	#
	# 5. 运行:
	# conda activate eval
	# cd eval_pack
	# bash eval/run_eval.sh
	#
	# 6. 结果会打印表格并保存到 eval/results/*.json
	#
	# ========== 目录结构 ==========
	#
	# eval_pack/
	# ├── eval/ <- 评测脚本
	# ├── models/ <- VoRA 模型代码
	# ├── tools/ <- merge_lora.py
	# ├── generation_files/ <- tokenizer/processor
	# ├── data/
	# │ ├── eval_vora.jsonl <- VoRA 系列用 (相对路径)
	# │ └── eval_qwenvl.jsonl <- Qwen-VL 系列用 (绝对路径)
	# ├── vora_merged_250/ <- VoRA merged (~32G)
	# ├── lora_merged/ <- Qwen-VL+LoRA merged (~7G)
	# ├── circulant_merged/ <- Qwen-VL+Circulant merged (~7G)
	# └── requirements_eval.txt
	#
	# =============================================================

	set -e
	cd "$(dirname "$0")/.."

	# ---------- 路径配置 (根据你的环境修改) ----------
	# 基线模型 (另一台机器上已有)
	QWEN_MODEL="/path/to/Qwen2.5-7B-Instruct"
	QWEN_VL_MODEL="/path/to/Qwen2.5-VL-3B-Instruct"
	AIMV2_PATH="/path/to/aimv2-huge-patch14-448"

	# 评测数据
	EVAL_DATA_VORA="data/eval_vora.jsonl" # VoRA 系列用 (相对路径 frames/xxx.jpg)
	EVAL_DATA_QWEN="data/eval_qwenvl.jsonl" # Qwen-VL 系列用 (绝对路径, 需要按目标机器修改)

	DTYPE="float16"
	RESULTS_DIR="eval/results"

	# 已 merge 好的模型
	VORA_MERGED="vora_merged_250"
	VORA_CIRC_MERGED="/path/to/vora-circulant-merged"
	LORA_MERGED="lora_merged"
	CIRC_MERGED="circulant_merged"
	# -----------------------------------------------

	mkdir -p "$RESULTS_DIR"

	pip install nltk rouge-score qwen-vl-utils 2>/dev/null \|\| true
	python -c "import nltk; nltk.download('punkt', quiet=True); nltk.download('punkt_tab', quiet=True)" 2>/dev/null \|\| true

	echo ""
	echo "=============================================="
	echo " Step 1: Qwen2.5-7B Text-Only Baseline"
	echo "=============================================="
	python eval/eval_qwen_baseline.py \
	--mode all \
	--model-path "$QWEN_MODEL" \
	--eval-data "$EVAL_DATA_QWEN" \
	--max-samples 200 \
	--max-new-tokens 256 \
	--dtype "$DTYPE" \
	--output "$RESULTS_DIR/qwen_baseline.json"

	echo ""
	echo "=============================================="
	echo " Step 2: VoRA Full Eval"
	echo "=============================================="
	python eval/eval_vora.py \
	--mode all \
	--checkpoint "$VORA_MERGED" \
	--merged \
	--eval-data "$EVAL_DATA_VORA" \
	--image-processor "$AIMV2_PATH" \
	--max-samples 200 \
	--max-new-tokens 256 \
	--dtype "$DTYPE" \
	--output "$RESULTS_DIR/vora_best.json"

	echo ""
	echo "=============================================="
	echo " Step 3: VoRA-Circulant Full Eval"
	echo "=============================================="
	python eval/eval_vora.py \
	--mode all \
	--checkpoint "$VORA_CIRC_MERGED" \
	--merged \
	--eval-data "$EVAL_DATA_VORA" \
	--image-processor "$AIMV2_PATH" \
	--max-samples 200 \
	--max-new-tokens 256 \
	--dtype "$DTYPE" \
	--output "$RESULTS_DIR/vora_circulant.json"

	echo ""
	echo "=============================================="
	echo " Step 4: Qwen2.5-VL-3B Original Baseline"
	echo "=============================================="
	python eval/eval_qwen_vl.py \
	--mode all \
	--model-path "$QWEN_VL_MODEL" \
	--eval-data "$EVAL_DATA_QWEN" \
	--max-samples 200 \
	--max-new-tokens 256 \
	--dtype "$DTYPE" \
	--output "$RESULTS_DIR/qwen_vl_original.json"

	echo ""
	echo "=============================================="
	echo " Step 5: Qwen2.5-VL-3B + LoRA"
	echo "=============================================="
	if [ -d "$LORA_MERGED" ]; then
	python eval/eval_qwen_vl.py \
	--mode all \
	--model-path "$LORA_MERGED" \
	--eval-data "$EVAL_DATA_QWEN" \
	--max-samples 200 \
	--max-new-tokens 256 \
	--dtype "$DTYPE" \
	--output "$RESULTS_DIR/qwen_vl_lora.json"
	else
	echo "LoRA merged model not found at $LORA_MERGED, skipping."
	fi

	echo ""
	echo "=============================================="
	echo " Step 6: Qwen2.5-VL-3B + Block-Circulant"
	echo "=============================================="
	if [ -d "$CIRC_MERGED" ]; then
	python eval/eval_qwen_vl.py \
	--mode all \
	--model-path "$CIRC_MERGED" \
	--eval-data "$EVAL_DATA_QWEN" \
	--max-samples 200 \
	--max-new-tokens 256 \
	--dtype "$DTYPE" \
	--output "$RESULTS_DIR/qwen_vl_circulant.json"
	else
	echo "Block-Circulant merged model not found at $CIRC_MERGED, skipping."
	fi

	echo ""
	echo "=============================================="
	echo " Summary"
	echo "=============================================="
	python -c "
	import json, os, math

	results_dir = '$RESULTS_DIR'

	print('='*70)
	print(f'{\"Model\":<35} {\"Loss\":>10} {\"PPL\":>8} {\"BLEU-4\":>8} {\"ROUGE-L\":>8}')
	print('-'*70)

	def print_row(name, filepath):
	if not os.path.exists(filepath):
	return
	r = json.load(open(filepath))
	ppl = r.get('perplexity', None)
	loss = math.log(ppl) if ppl and ppl != float('inf') else None
	b4 = r.get('BLEU-4', None)
	rl = r.get('ROUGE-L', None)
	loss_s = f'{loss:.4f}' if loss else '-'
	ppl_s = f'{ppl:.2f}' if ppl else '-'
	b4_s = f'{b4:.4f}' if b4 else '-'
	rl_s = f'{rl:.4f}' if rl else '-'
	print(f'{name:<35} {loss_s:>10} {ppl_s:>8} {b4_s:>8} {rl_s:>8}')

	print_row('Qwen2.5-7B (text-only)', os.path.join(results_dir, 'qwen_baseline.json'))
	print_row('VoRA', os.path.join(results_dir, 'vora_best.json'))
	print_row('VoRA-Circulant', os.path.join(results_dir, 'vora_circulant.json'))

	print('-'*70)

	print_row('Qwen2.5-VL-3B (original)', os.path.join(results_dir, 'qwen_vl_original.json'))
	print_row('Qwen2.5-VL-3B + LoRA', os.path.join(results_dir, 'qwen_vl_lora.json'))
	print_row('Qwen2.5-VL-3B + Circulant', os.path.join(results_dir, 'qwen_vl_circulant.json'))

	print('='*70)
	"

	echo ""
	echo "All results saved to \$RESULTS_DIR/"
	echo "Done!"