GroundFlow / scripts /eval_32b_ours_conf010.sh

32B full results: nips.tex (Vanilla/SD-RPN/GroundFlow), trainer_state for run2 + SD-RPN, textvqa patch tile figures (PNG), eval/sync scripts

d6e1c8a verified 6 months ago

Raw History Blame Contribute Delete

2.88 kB

#!/usr/bin/env bash
# 32B GroundFlow (run1, α=0.667 τ=1.5) at conf=0.10 — ablation of ROI confidence threshold.
# All 10 main benchmarks. enable_iwa=False already in saved config.
# Layout: 5 idle GPUs (0,1,2,4,5), staggered 90s to avoid NFS thrash.
set -u

REPO=/opt/tiger/thothvl_pretrain
M=/mnt/bn/leonworkspace/terry/model
R=/mnt/bn/leonworkspace/terry/results
NAS_LOGS=/mnt/bn/leonworkspace/terry/logs
LOG_DIR=$NAS_LOGS/eval_32b_ours_conf010_$(date +%Y%m%d_%H%M)
mkdir -p "$LOG_DIR"
SUMMARY="$LOG_DIR/summary.log"
: > "$SUMMARY"

cd "$REPO/lmms-eval"
export PYTHONPATH=$REPO/QWENVL-PRIVATE:$PYTHONPATH
export HF_TOKEN=<HF_TOKEN>
export HF_HOME=/mnt/bn/leonworkspace/HF_HOME
export HF_DATASETS_CACHE=$HF_HOME/datasets
export OPENAI_API_KEY=<OPENAI_API_KEY>
export OPENAI_API_URL=https://search-va.byteintl.net/gpt/openapi/online/v2/crawl
export MODEL_VERSION=gpt-4o-2024-11-20
unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY

CKPT=$M/qwen3vl-32b-dense-roi-K49T3-150k-confluent-a0.667-t1.5-iwa-0505_0321
ATTN=flash_attention_2
TAG=publish-32b-ours-576-conf0.10
CONF=0.10

ours_args() { echo "pretrained=$CKPT,device_map=auto,two_stage_roi=True,roi_baseline=True,roi_conf_thresh=$CONF,high_res_thresh=0.1,attn_implementation=$ATTN"; }

run_one() {
    local gpu=$1 task=$2
    local out_dir="$R/$TAG/$task"
    if find "$out_dir" -name "*results.json" 2>/dev/null | grep -q .; then
        echo "[$(date '+%F %T')] SKIP gpu$gpu $TAG/$task (done)" | tee -a "$SUMMARY"
        return
    fi
    mkdir -p "$out_dir"
    local log="$LOG_DIR/${task}_gpu${gpu}.log"
    echo "[$(date '+%F %T')] START gpu$gpu $TAG/$task" | tee -a "$SUMMARY"
    CUDA_VISIBLE_DEVICES=$gpu python3 -m lmms_eval \
        --model qwen3_vl_hybrid --model_args "$(ours_args)" \
        --tasks "$task" --batch_size 1 \
        --output_path "$out_dir" \
        --log_samples --log_samples_suffix "$TAG" \
        > "$log" 2>&1
    local rc=$?
    if [ $rc -eq 0 ] && find "$out_dir" -name "*results.json" 2>/dev/null | grep -q .; then
        echo "[$(date '+%F %T')] DONE  gpu$gpu $TAG/$task" | tee -a "$SUMMARY"
    else
        echo "[$(date '+%F %T')] ERR   gpu$gpu $TAG/$task rc=$rc — $log" | tee -a "$SUMMARY"
    fi
}

queue_gpu0() { run_one 0 mmerealworld; }                                              # 23k
queue_gpu1() { run_one 1 docvqa_val; }                                                # 5349
queue_gpu2() { run_one 2 textvqa_val; }                                               # 5000
queue_gpu4() { run_one 4 chartqa; run_one 4 hrbench4k; run_one 4 hrbench8k; }
queue_gpu5() { run_one 5 infovqa_val; run_one 5 ocrbench; run_one 5 realworldqa; run_one 5 vstar_bench; }

queue_gpu0 &
sleep 90; queue_gpu1 &
sleep 90; queue_gpu2 &
sleep 90; queue_gpu4 &
sleep 90; queue_gpu5 &
wait

echo "[$(date '+%F %T')] ALL conf=0.10 EVAL DONE" | tee -a "$SUMMARY"
echo "Logs: $LOG_DIR"

	#!/usr/bin/env bash
	# 32B GroundFlow (run1, α=0.667 τ=1.5) at conf=0.10 — ablation of ROI confidence threshold.
	# All 10 main benchmarks. enable_iwa=False already in saved config.
	# Layout: 5 idle GPUs (0,1,2,4,5), staggered 90s to avoid NFS thrash.
	set -u

	REPO=/opt/tiger/thothvl_pretrain
	M=/mnt/bn/leonworkspace/terry/model
	R=/mnt/bn/leonworkspace/terry/results
	NAS_LOGS=/mnt/bn/leonworkspace/terry/logs
	LOG_DIR=$NAS_LOGS/eval_32b_ours_conf010_$(date +%Y%m%d_%H%M)
	mkdir -p "$LOG_DIR"
	SUMMARY="$LOG_DIR/summary.log"
	: > "$SUMMARY"

	cd "$REPO/lmms-eval"
	export PYTHONPATH=$REPO/QWENVL-PRIVATE:$PYTHONPATH
	export HF_TOKEN=<HF_TOKEN>
	export HF_HOME=/mnt/bn/leonworkspace/HF_HOME
	export HF_DATASETS_CACHE=$HF_HOME/datasets
	export OPENAI_API_KEY=<OPENAI_API_KEY>
	export OPENAI_API_URL=https://search-va.byteintl.net/gpt/openapi/online/v2/crawl
	export MODEL_VERSION=gpt-4o-2024-11-20
	unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY

	CKPT=$M/qwen3vl-32b-dense-roi-K49T3-150k-confluent-a0.667-t1.5-iwa-0505_0321
	ATTN=flash_attention_2
	TAG=publish-32b-ours-576-conf0.10
	CONF=0.10

	ours_args() { echo "pretrained=$CKPT,device_map=auto,two_stage_roi=True,roi_baseline=True,roi_conf_thresh=$CONF,high_res_thresh=0.1,attn_implementation=$ATTN"; }

	run_one() {
	local gpu=$1 task=$2
	local out_dir="$R/$TAG/$task"
	if find "$out_dir" -name "*results.json" 2>/dev/null \| grep -q .; then
	echo "[$(date '+%F %T')] SKIP gpu$gpu $TAG/$task (done)" \| tee -a "$SUMMARY"
	return
	fi
	mkdir -p "$out_dir"
	local log="$LOG_DIR/${task}_gpu${gpu}.log"
	echo "[$(date '+%F %T')] START gpu$gpu $TAG/$task" \| tee -a "$SUMMARY"
	CUDA_VISIBLE_DEVICES=$gpu python3 -m lmms_eval \
	--model qwen3_vl_hybrid --model_args "$(ours_args)" \
	--tasks "$task" --batch_size 1 \
	--output_path "$out_dir" \
	--log_samples --log_samples_suffix "$TAG" \
	> "$log" 2>&1
	local rc=$?
	if [ $rc -eq 0 ] && find "$out_dir" -name "*results.json" 2>/dev/null \| grep -q .; then
	echo "[$(date '+%F %T')] DONE gpu$gpu $TAG/$task" \| tee -a "$SUMMARY"
	else
	echo "[$(date '+%F %T')] ERR gpu$gpu $TAG/$task rc=$rc — $log" \| tee -a "$SUMMARY"
	fi
	}

	queue_gpu0() { run_one 0 mmerealworld; } # 23k
	queue_gpu1() { run_one 1 docvqa_val; } # 5349
	queue_gpu2() { run_one 2 textvqa_val; } # 5000
	queue_gpu4() { run_one 4 chartqa; run_one 4 hrbench4k; run_one 4 hrbench8k; }
	queue_gpu5() { run_one 5 infovqa_val; run_one 5 ocrbench; run_one 5 realworldqa; run_one 5 vstar_bench; }

	queue_gpu0 &
	sleep 90; queue_gpu1 &
	sleep 90; queue_gpu2 &
	sleep 90; queue_gpu4 &
	sleep 90; queue_gpu5 &
	wait

	echo "[$(date '+%F %T')] ALL conf=0.10 EVAL DONE" \| tee -a "$SUMMARY"
	echo "Logs: $LOG_DIR"