# 基于LAMRA实现拍立淘同款embed模型训练 set -x ## 分布式配置,一个节点一张卡 NUM_GPUS=1 DISTRIBUTED_ARGS=" --nnodes=1 \ --nproc_per_node ${NUM_GPUS} \ --rdzv_backend c10d \ --rdzv_endpoint localhost:0 \ " ## 模型设置 ### backbone proj模型设置 MODEL_ID=tbstars_ecom_emb_3b MODEL_LOCAL_PATH=/data/oss_bucket_0/peifeng/vlm2vec/models/tbstar/TBStars_VL_Ecom_EMB_3B/ PROJ_NAME=load_add_projector EMBED_DIM=512 ### parallel scale setting PARSCALE_N=1 PARSCALE_N_TOKENS=0 PARSCALE_ATTN_SMOOTH=0.01 TRAIN_PARSCALE=False ## 数据集设置 TABLES="odps://pailitao_search_offline/tables/plt_lzy_rank_tuwen_zhaoxiangsi_train/ds=20260617" MAX_LINES=10000 IMAGE_TYPE="both" TEXT_REQUIRE=True ## 训练设置 ### 训练参数设置 TRAIN_VISION_ENCODER=True USE_VISION_LORA=False TRAIN_VISION_PROJECTOR=True TRAIN_LLM=True USE_LORA=True Q_LORA=False LORA_R=128 LORA_ALPHA=256 ### DS和基本训练参数设置 DS_STAGE=zero2 PER_DEVICE_BATCH_SIZE=4 GRAD_ACCUM=1 NUM_EPOCHS=5 SAVE_STEPS=200 LR=1e-5 MODEL_MAX_LEN=512 ## 保存和读取设置 OUTPUT_DIR=/data/oss_bucket_0/peifeng/multi_vlm2vec/multi_vlm2vec_train/test_train_2607/ RUN_ID=tbstars_train_debug PROJ_PRETRAINED_PTH=None RESUME=False ## 除了可以给定False或者True,也可以直接给定读取的地址,其中当没有ckpt保存在其中的时候,就选择False echo "begin training!" torchrun $DISTRIBUTED_ARGS train/train_tuwen_tbstars_infonce.py \ --model_id $MODEL_ID \ --tables "${TABLES}" \ --max_lines $MAX_LINES \ --output_dir ${OUTPUT_DIR}/${RUN_ID} \ --report_to tensorboard \ --run_name $RUN_ID \ --deepspeed ./ds_configs/${DS_STAGE}.json \ --bf16 True \ --num_train_epochs $NUM_EPOCHS \ --per_device_train_batch_size $PER_DEVICE_BATCH_SIZE \ --per_device_eval_batch_size 12 \ --gradient_accumulation_steps $GRAD_ACCUM \ --eval_strategy "no" \ --save_strategy "steps" \ --save_total_limit 200 \ --learning_rate ${LR} \ --weight_decay 0. \ --warmup_ratio 0.03 \ --lr_scheduler_type "constant_with_warmup" \ --logging_steps 10 \ --tf32 True \ --model_max_length $MODEL_MAX_LEN \ --gradient_checkpointing False \ --dataloader_num_workers 0 \ --train_vision_encoder $TRAIN_VISION_ENCODER \ --use_vision_lora $USE_VISION_LORA \ --train_vision_projector $TRAIN_VISION_PROJECTOR \ --use_lora $USE_LORA \ --q_lora $Q_LORA \ --lora_r $LORA_R \ --lora_alpha $LORA_ALPHA \ --model_local_path $MODEL_LOCAL_PATH \ --use_flash_attn True \ --save_steps $SAVE_STEPS \ --stage classify \ --embed_dim ${EMBED_DIM} \ --model_name_or_path_proj $PROJ_PRETRAINED_PTH \ --resume_from_checkpoint $RESUME \ --proj_name $PROJ_NAME \ --parscale_n $PARSCALE_N \ --parscale_n_tokens $PARSCALE_N_TOKENS \ --parscale_attn_smooth $PARSCALE_ATTN_SMOOTH \ --image_type $IMAGE_TYPE \ --text_require $TEXT_REQUIRE \ --train_parscale $TRAIN_PARSCALE \ --train_llm $TRAIN_LLM