liqi / code /rerank_ins /scripts /train /train_qwen35.sh
WaitXian's picture
Upload liqi files (part 2)
01cf98b verified
Raw History Blame Contribute Delete
2.39 kB
#!/bin/bash
# ============ Configuration ============
MODEL_ID=qwen3_5-9b
MODEL_LOCAL_PATH=/mnt/workspace/siku/Qwen3.5-9B # TODO: set your model path
TRAIN_DATA_PATH=/data/oss_bucket_0/siku/Last_reranker/dataset/all_data_wo_q_bbox/00-07/with_bbox/train_format/train.jsonl # TODO: set your train data path
EVAL_DATA_PATH=/data/oss_bucket_0/siku/Last_reranker/dataset/all_data_wo_q_bbox/00-07/with_bbox/train_format/eval.jsonl # TODO: set your eval data path
TRAIN_VISION_ENCODER=False
USE_VISION_LORA=False
TRAIN_VISION_PROJECTOR=False
USE_LORA=False
Q_LORA=False
LORA_R=128
LORA_ALPHA=128
RUN_ID=${MODEL_ID}-reranker
DS_STAGE=zero2
PER_DEVICE_BATCH_SIZE=2
GRAD_ACCUM=2
NUM_EPOCHS=5
LR=1e-5
MODEL_MAX_LEN=8192
NUM_TRAIN_DOCS=10
FULL_CROP_MODE=True
FORMAT_PROMPT=format_prompt/listwise_reranker_v2.jinja
OUTPUT_DIR=/data/oss_bucket_0/rerank_9B/output/$RUN_ID # TODO : 修改路径
# ============ Launch ============
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 torchrun --nproc_per_node=8 train/train_reranker_qwen35.py \
--model_id $MODEL_ID \
--output_dir $OUTPUT_DIR \
--report_to tensorboard \
--run_name $RUN_ID \
--deepspeed ./ds_configs/${DS_STAGE}.json \
--bf16 True \
--num_train_epochs $NUM_EPOCHS \
--per_device_train_batch_size $PER_DEVICE_BATCH_SIZE \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps $GRAD_ACCUM \
--eval_strategy "no" \
--eval_steps 100 \
--eval_accumulation_steps 4 \
--save_strategy "steps" \
--save_steps 100 \
--save_total_limit 500 \
--learning_rate ${LR} \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length $MODEL_MAX_LEN \
--gradient_checkpointing True \
--dataloader_num_workers 8 \
--train_vision_encoder $TRAIN_VISION_ENCODER \
--use_vision_lora $USE_VISION_LORA \
--train_vision_projector $TRAIN_VISION_PROJECTOR \
--use_lora $USE_LORA \
--q_lora $Q_LORA \
--lora_r $LORA_R \
--lora_alpha $LORA_ALPHA \
--model_local_path $MODEL_LOCAL_PATH \
--train_data_path $TRAIN_DATA_PATH \
--eval_data_path $EVAL_DATA_PATH \
--use_flash_attn False \
--format_prompt $FORMAT_PROMPT \
--num_train_docs $NUM_TRAIN_DOCS \
--full_crop_mode $FULL_CROP_MODE \
--max_full_tokens 160 \
--max_crop_tokens 160