Download code/rerank_ins/cot_gen/run_gen_cot.sh from WaitXian/liqi: direct link, hf CLI and curl.
- Browser
- Download file 1.03 kB
-
https://huggingface.co/WaitXian/liqi/resolve/main/code/rerank_ins/cot_gen/run_gen_cot.sh
- Command line
-
hf download hf://WaitXian/liqi/code/rerank_ins/cot_gen/run_gen_cot.sh
-
curl -L -o run_gen_cot.sh https://huggingface.co/WaitXian/liqi/resolve/main/code/rerank_ins/cot_gen/run_gen_cot.sh
1.03 kB
| # 用本地大 Qwen-VL 模型为训练集批量生成 CoT 数据 | |
| MODEL_PATH=/data/oss_bucket_0/planck/hf_models/Qwen3.6-27B # TODO: 本地大模型路径 | |
| TRAIN_DATA_PATH=/data/oss_bucket_0/siku/Last_reranker/dataset/all_data_wo_q_bbox/00-07/with_bbox/train_format/train.jsonl | |
| OUTPUT_PATH=./cot_data/train_cot.jsonl | |
| # 多机/多进程分片时修改 SHARD_ID / NUM_SHARDS,各分片写不同 OUTPUT_PATH 后合并 | |
| SHARD_ID=0 | |
| NUM_SHARDS=1 | |
| # 试跑:设为 "--test 20" 只生成前 20 条(写到 train_cot.test.jsonl),留空则全量生成 | |
| TEST="--test 10" | |
| python cot_gen/gen_cot_data.py \ | |
| --model-path $MODEL_PATH \ | |
| --train-data-path $TRAIN_DATA_PATH \ | |
| --output-path $OUTPUT_PATH \ | |
| --num-docs 10 \ | |
| --seed 42 \ | |
| --max-crop-tokens 80 \ | |
| --patch-size 16 \ | |
| --tensor-parallel-size 4 \ | |
| --max-model-len 16384 \ | |
| --max-new-tokens 4096 \ | |
| --max-retries 3 \ | |
| --retry-temperature 0.7 \ | |
| --batch-size 64 \ | |
| --shard-id $SHARD_ID \ | |
| --num-shards $NUM_SHARDS \ | |
| $TEST | |