Download code/multi_vlm2vec_train/scripts/train/train_local_tbstars.sh from WaitXian/liqi: direct link, hf CLI and curl.
- Browser
- Download file 3.47 kB
-
https://huggingface.co/WaitXian/liqi/resolve/main/code/multi_vlm2vec_train/scripts/train/train_local_tbstars.sh
- Command line
-
hf download hf://WaitXian/liqi/code/multi_vlm2vec_train/scripts/train/train_local_tbstars.sh
-
curl -L -o train_local_tbstars.sh https://huggingface.co/WaitXian/liqi/resolve/main/code/multi_vlm2vec_train/scripts/train/train_local_tbstars.sh
3.47 kB
| # 基于LAMRA实现拍立淘同款embed模型训练 | |
| set -x | |
| ## 分布式配置,一个节点一张卡 | |
| NUM_GPUS=1 | |
| DISTRIBUTED_ARGS=" | |
| --nnodes=1 \ | |
| --nproc_per_node ${NUM_GPUS} \ | |
| --rdzv_backend c10d \ | |
| --rdzv_endpoint localhost:0 \ | |
| " | |
| ## 模型设置 | |
| ### backbone proj模型设置 | |
| MODEL_ID=tbstars_ecom_emb_3b | |
| MODEL_LOCAL_PATH=/data/oss_bucket_0/peifeng/vlm2vec/models/tbstar/TBStars_VL_Ecom_EMB_3B/ | |
| PROJ_NAME=load_add_projector | |
| EMBED_DIM=512 | |
| ### parallel scale setting | |
| PARSCALE_N=1 | |
| PARSCALE_N_TOKENS=0 | |
| PARSCALE_ATTN_SMOOTH=0.01 | |
| TRAIN_PARSCALE=False | |
| ## 数据集设置 | |
| TABLES="odps://pailitao_search_offline/tables/plt_lzy_rank_tuwen_zhaoxiangsi_train/ds=20260617" | |
| MAX_LINES=10000 | |
| IMAGE_TYPE="both" | |
| TEXT_REQUIRE=True | |
| ## 训练设置 | |
| ### 训练参数设置 | |
| TRAIN_VISION_ENCODER=True | |
| USE_VISION_LORA=False | |
| TRAIN_VISION_PROJECTOR=True | |
| TRAIN_LLM=True | |
| USE_LORA=True | |
| Q_LORA=False | |
| LORA_R=128 | |
| LORA_ALPHA=256 | |
| ### DS和基本训练参数设置 | |
| DS_STAGE=zero2 | |
| PER_DEVICE_BATCH_SIZE=4 | |
| GRAD_ACCUM=1 | |
| NUM_EPOCHS=5 | |
| SAVE_STEPS=200 | |
| LR=1e-5 | |
| MODEL_MAX_LEN=512 | |
| ## 保存和读取设置 | |
| OUTPUT_DIR=/data/oss_bucket_0/peifeng/multi_vlm2vec/multi_vlm2vec_train/test_train_2607/ | |
| RUN_ID=tbstars_train_debug | |
| PROJ_PRETRAINED_PTH=None | |
| RESUME=False ## 除了可以给定False或者True,也可以直接给定读取的地址,其中当没有ckpt保存在其中的时候,就选择False | |
| echo "begin training!" | |
| torchrun $DISTRIBUTED_ARGS train/train_tuwen_tbstars_infonce.py \ | |
| --model_id $MODEL_ID \ | |
| --tables "${TABLES}" \ | |
| --max_lines $MAX_LINES \ | |
| --output_dir ${OUTPUT_DIR}/${RUN_ID} \ | |
| --report_to tensorboard \ | |
| --run_name $RUN_ID \ | |
| --deepspeed ./ds_configs/${DS_STAGE}.json \ | |
| --bf16 True \ | |
| --num_train_epochs $NUM_EPOCHS \ | |
| --per_device_train_batch_size $PER_DEVICE_BATCH_SIZE \ | |
| --per_device_eval_batch_size 12 \ | |
| --gradient_accumulation_steps $GRAD_ACCUM \ | |
| --eval_strategy "no" \ | |
| --save_strategy "steps" \ | |
| --save_total_limit 200 \ | |
| --learning_rate ${LR} \ | |
| --weight_decay 0. \ | |
| --warmup_ratio 0.03 \ | |
| --lr_scheduler_type "constant_with_warmup" \ | |
| --logging_steps 10 \ | |
| --tf32 True \ | |
| --model_max_length $MODEL_MAX_LEN \ | |
| --gradient_checkpointing False \ | |
| --dataloader_num_workers 0 \ | |
| --train_vision_encoder $TRAIN_VISION_ENCODER \ | |
| --use_vision_lora $USE_VISION_LORA \ | |
| --train_vision_projector $TRAIN_VISION_PROJECTOR \ | |
| --use_lora $USE_LORA \ | |
| --q_lora $Q_LORA \ | |
| --lora_r $LORA_R \ | |
| --lora_alpha $LORA_ALPHA \ | |
| --model_local_path $MODEL_LOCAL_PATH \ | |
| --use_flash_attn True \ | |
| --save_steps $SAVE_STEPS \ | |
| --stage classify \ | |
| --embed_dim ${EMBED_DIM} \ | |
| --model_name_or_path_proj $PROJ_PRETRAINED_PTH \ | |
| --resume_from_checkpoint $RESUME \ | |
| --proj_name $PROJ_NAME \ | |
| --parscale_n $PARSCALE_N \ | |
| --parscale_n_tokens $PARSCALE_N_TOKENS \ | |
| --parscale_attn_smooth $PARSCALE_ATTN_SMOOTH \ | |
| --image_type $IMAGE_TYPE \ | |
| --text_require $TEXT_REQUIRE \ | |
| --train_parscale $TRAIN_PARSCALE \ | |
| --train_llm $TRAIN_LLM |