#!/bin/bash 
configure_id=$1
model=$2
# output_dir=$3
task=$3
dir=$4
# task=hellaswag,rte,mmlu
echo "${configure_id} ${task}"
lm_eval --model hf \
    --model_args pretrained=${model} \
    --tasks $task \
    --device cuda:0 \
    --batch_size 16 \
    --output_path results/${dir}/${configure_id}.json