| # export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 | |
| # export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 | |
| # export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 | |
| # export no_proxy="$no_proxy,.byteintl.net" | |
| # export HF_ENDPOINT=https://hf-mirror.com | |
| model=$1 | |
| output_dir=$2 | |
| cuda_id=$3 | |
| model_name=$(basename "$model") | |
| output_dir="results3/${model_name}/${output_dir}" | |
| # model=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen/Qwen2.5-14B | |
| # model_name=$(basename "$model") | |
| mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 | |
| # 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） | |
| declare -A task_fewshot=( | |
| ["winogrande"]="5" | |
| ["truthfulqa_mc1"]="0" | |
| ["arc_challenge"]="25" | |
| ["hellaswag"]="10" | |
| # ["mmlu"]="5" | |
| # ["triviaqa"]="0" | |
| ["piqa"]="0" | |
| ["boolq"]="0" | |
| ) | |
| # 4. 循环执行所有任务（重复逻辑一次性写死） | |
| for task in "${!task_fewshot[@]}"; do | |
| num_fewshot=${task_fewshot[$task]} | |
| output_path="${output_dir}/${task}.json" | |
| # 打印进度信息 | |
| echo -e "\n==================================================" | |
| echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" | |
| echo "输出路径：$output_path" | |
| echo "==================================================" | |
| # 核心评估命令（仅替换动态变量） | |
| CUDA_VISIBLE_DEVICES=2,3,6,7 \ | |
| accelerate launch --num_processes=4 --main_process_port 1445 \ | |
| lm_eval --model hf \ | |
| --model_args "pretrained=${model}" \ | |
| --tasks "$task" \ | |
| --device cuda \ | |
| --batch_size auto \ | |
| --num_fewshot "$num_fewshot" \ | |
| --output_path "$output_path" | |
| # 检查命令是否执行成功 | |
| if [ $? -eq 0 ]; then | |
| echo "任务 $task 评估完成！" | |
| else | |
| echo "警告：任务 $task 执行失败！" | |
| fi | |
| done | |