liqi / code /multi_vlm2vec_train /scripts /train /train_local_tbstars.sh
WaitXian's picture
Upload liqi files
0ab2566 verified
Raw History Blame Contribute Delete
3.47 kB
# 基于LAMRA实现拍立淘同款embed模型训练
set -x
## 分布式配置,一个节点一张卡
NUM_GPUS=1
DISTRIBUTED_ARGS="
--nnodes=1 \
--nproc_per_node ${NUM_GPUS} \
--rdzv_backend c10d \
--rdzv_endpoint localhost:0 \
"
## 模型设置
### backbone proj模型设置
MODEL_ID=tbstars_ecom_emb_3b
MODEL_LOCAL_PATH=/data/oss_bucket_0/peifeng/vlm2vec/models/tbstar/TBStars_VL_Ecom_EMB_3B/
PROJ_NAME=load_add_projector
EMBED_DIM=512
### parallel scale setting
PARSCALE_N=1
PARSCALE_N_TOKENS=0
PARSCALE_ATTN_SMOOTH=0.01
TRAIN_PARSCALE=False
## 数据集设置
TABLES="odps://pailitao_search_offline/tables/plt_lzy_rank_tuwen_zhaoxiangsi_train/ds=20260617"
MAX_LINES=10000
IMAGE_TYPE="both"
TEXT_REQUIRE=True
## 训练设置
### 训练参数设置
TRAIN_VISION_ENCODER=True
USE_VISION_LORA=False
TRAIN_VISION_PROJECTOR=True
TRAIN_LLM=True
USE_LORA=True
Q_LORA=False
LORA_R=128
LORA_ALPHA=256
### DS和基本训练参数设置
DS_STAGE=zero2
PER_DEVICE_BATCH_SIZE=4
GRAD_ACCUM=1
NUM_EPOCHS=5
SAVE_STEPS=200
LR=1e-5
MODEL_MAX_LEN=512
## 保存和读取设置
OUTPUT_DIR=/data/oss_bucket_0/peifeng/multi_vlm2vec/multi_vlm2vec_train/test_train_2607/
RUN_ID=tbstars_train_debug
PROJ_PRETRAINED_PTH=None
RESUME=False ## 除了可以给定False或者True,也可以直接给定读取的地址,其中当没有ckpt保存在其中的时候,就选择False
echo "begin training!"
torchrun $DISTRIBUTED_ARGS train/train_tuwen_tbstars_infonce.py \
--model_id $MODEL_ID \
--tables "${TABLES}" \
--max_lines $MAX_LINES \
--output_dir ${OUTPUT_DIR}/${RUN_ID} \
--report_to tensorboard \
--run_name $RUN_ID \
--deepspeed ./ds_configs/${DS_STAGE}.json \
--bf16 True \
--num_train_epochs $NUM_EPOCHS \
--per_device_train_batch_size $PER_DEVICE_BATCH_SIZE \
--per_device_eval_batch_size 12 \
--gradient_accumulation_steps $GRAD_ACCUM \
--eval_strategy "no" \
--save_strategy "steps" \
--save_total_limit 200 \
--learning_rate ${LR} \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "constant_with_warmup" \
--logging_steps 10 \
--tf32 True \
--model_max_length $MODEL_MAX_LEN \
--gradient_checkpointing False \
--dataloader_num_workers 0 \
--train_vision_encoder $TRAIN_VISION_ENCODER \
--use_vision_lora $USE_VISION_LORA \
--train_vision_projector $TRAIN_VISION_PROJECTOR \
--use_lora $USE_LORA \
--q_lora $Q_LORA \
--lora_r $LORA_R \
--lora_alpha $LORA_ALPHA \
--model_local_path $MODEL_LOCAL_PATH \
--use_flash_attn True \
--save_steps $SAVE_STEPS \
--stage classify \
--embed_dim ${EMBED_DIM} \
--model_name_or_path_proj $PROJ_PRETRAINED_PTH \
--resume_from_checkpoint $RESUME \
--proj_name $PROJ_NAME \
--parscale_n $PARSCALE_N \
--parscale_n_tokens $PARSCALE_N_TOKENS \
--parscale_attn_smooth $PARSCALE_ATTN_SMOOTH \
--image_type $IMAGE_TYPE \
--text_require $TEXT_REQUIRE \
--train_parscale $TRAIN_PARSCALE \
--train_llm $TRAIN_LLM