cot_generate/start_server.sh

#!/bin/bash
# ============================================================
# 启动 Qwen3.5-122B-A10B vLLM 推理服务
# 用于视频 CoT 数据生成
#
# 使用方式: bash start_server.sh
# 启动后会在 localhost:8000 提供 OpenAI 兼容 API
# ============================================================

# --- 可调参数 ---
MODEL="/root/zwj/Model/Qwen3.5-122B-A10B"
PORT=8000
TP_SIZE=8                        # tensor parallel, 8卡 × 96GB = 768GB, 模型~250GB
GPU_MEM_UTIL=0.92                # GPU显存利用率, 尽量调高给KV Cache留空间
MAX_MODEL_LEN=131072             # 最大序列长度 (模型上下文上限128K, 覆盖~161帧×768tokens)
MAX_NUM_SEQS=8                   # 最大并发序列数

# --- 视频处理参数 ---
export VIDEO_MAX_TOKEN_NUM=768   # 视频每帧token数, 高分辨率以准确读取时间戳水印
export FPS_MAX_FRAMES=32         # 最大采样帧数
export VIDEO_MAX_PIXELS=602112   # 视频帧最大像素, 与 768 tokens 匹配 (768 × 784)
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7

echo "============================================"
echo " Starting Qwen3.5-122B-A10B vLLM Server"
echo " Model: ${MODEL}"
echo " GPUs: ${CUDA_VISIBLE_DEVICES} (TP=${TP_SIZE})"
echo " Port: ${PORT}"
echo " Max Model Len: ${MAX_MODEL_LEN}"
echo " Max Num Seqs: ${MAX_NUM_SEQS}"
echo " Video Frames: ${FPS_MAX_FRAMES}, Tokens/Frame: ${VIDEO_MAX_TOKEN_NUM}"
echo "============================================"

swift deploy \
    --model ${MODEL} \
    --infer_backend vllm \
    --vllm_tensor_parallel_size ${TP_SIZE} \
    --vllm_gpu_memory_utilization ${GPU_MEM_UTIL} \
    --vllm_max_model_len ${MAX_MODEL_LEN} \
    --vllm_max_num_seqs ${MAX_NUM_SEQS} \
    --enable_thinking true \
    --port ${PORT}


# tmux new-session -d -s vllm_server "cd /root/zwj/CoT_generate && bash start_server.sh" 2>&1