vLLM_VLM_OCR_Nginx / scripts /preload_models.sh
GitLab CI
Deploy commit 2c06a658 via GitLab CI
aef2876
Raw History Blame Contribute Delete
2.85 kB
#!/bin/bash
# Preload the 3 models from the HF Hub into LOCAL disk (HF_HOME=/tmp/hf) using
# hf_xet (chunked, parallel, fast) BEFORE starting vLLM.
# Why: reading weights through the FUSE-mounted bucket (/data) is ~12 Mo/s
# (checkpoint loads took >10 min); a direct HF Hub download is ~100x faster,
# and loading from local disk afterwards is instant (page cache).
# /tmp is ephemeral, so this re-downloads at each Space restart (~1-3 min),
# which is still far better than FUSE. The /data bucket stays mounted for
# other persistent data.
set -euo pipefail
# hf_xet is the default transfer backend in huggingface_hub 2.x; this enables its
# high-performance mode (chunked parallel downloads). HF_HUB_ENABLE_HF_TRANSFER is
# deprecated in hub 2.x.
export HF_XET_HIGH_PERFORMANCE=1
HF_HOME="${HF_HOME:-/tmp/hf}"
mkdir -p "$HF_HOME"
for m in "${MODEL_OCR:-zai-org/GLM-OCR}" "${MODEL_VLM:-Qwen/Qwen3-VL-4B-Instruct}" "${LAYOUT_MODEL:-PaddlePaddle/PP-DocLayoutV3_onnx}"; do
echo "[preload] downloading $m -> $HF_HOME (hf_xet)"
# huggingface-cli no longer exists in huggingface_hub 2.x -> use the Python API
python3 - "$m" <<'PYEOF'
import sys
from huggingface_hub import snapshot_download
snapshot_download(sys.argv[1], ignore_patterns=["*.md", "*.gguf", "*.bin"])
PYEOF
done
# Cap the max input image AREA of BOTH VLMs to MAX_IMAGE_PX (default 2250000 =
# 1500x1500). Notes:
# - 'longest_edge' is a total pixel AREA (vLLM uses it as max_pixels), NOT an
# edge length. Qwen3-VL ships 16777216 (16 Mpx = 16384 vision tokens),
# GLM-OCR ships 9633792 (9.6 Mpx = ~12288 tokens).
# - The V1 engine profiles the vision encoder at startup with a dummy item of
# MAXIMUM feature size, so a large cap = large activation peak (this crashed
# vllm-vlm on T4 with 'Available KV cache memory: -0.27 GiB').
# - Input contract of this Space: one A4 page per request, pre-resized to at
# most 1200x1200 (1.44 Mpx). The 2.25 Mpx cap therefore never triggers on
# real inputs; it only bounds the worst case: ~2197 vision tokens for
# Qwen3-VL (1024 px/token), ~2870 for GLM-OCR (784 px/token), leaving >1000
# tokens of the 4096 context for prompt + generation.
MAX_IMAGE_PX="${MAX_IMAGE_PX:-2250000}"
for PC in $(find "$HF_HOME" \( -path "*GLM-OCR*" -o -path "*Qwen3-VL*" \) -name preprocessor_config.json 2>/dev/null); do
python3 - "$PC" "$MAX_IMAGE_PX" <<'EOF'
import json, sys
p, cap = sys.argv[1], int(sys.argv[2])
d = json.load(open(p))
sz = d.get("size", {})
if "longest_edge" in sz:
old = int(sz["longest_edge"])
sz["longest_edge"] = min(old, cap)
d["size"] = sz
json.dump(d, open(p, "w"), indent=2)
print(f"[preload] capped size.longest_edge in {p}: {old} -> {sz['longest_edge']}")
else:
print(f"[preload] WARNING: no size.longest_edge in {p}")
EOF
done
echo "[preload] all models cached in $HF_HOME"