#!/bin/bash # Preload the 3 models from the HF Hub into LOCAL disk (HF_HOME=/tmp/hf) using # hf_xet (chunked, parallel, fast) BEFORE starting vLLM. # Why: reading weights through the FUSE-mounted bucket (/data) is ~12 Mo/s # (checkpoint loads took >10 min); a direct HF Hub download is ~100x faster, # and loading from local disk afterwards is instant (page cache). # /tmp is ephemeral, so this re-downloads at each Space restart (~1-3 min), # which is still far better than FUSE. The /data bucket stays mounted for # other persistent data. set -euo pipefail # hf_xet is the default transfer backend in huggingface_hub 2.x; this enables its # high-performance mode (chunked parallel downloads). HF_HUB_ENABLE_HF_TRANSFER is # deprecated in hub 2.x. export HF_XET_HIGH_PERFORMANCE=1 HF_HOME="${HF_HOME:-/tmp/hf}" mkdir -p "$HF_HOME" for m in "${MODEL_OCR:-zai-org/GLM-OCR}" "${MODEL_VLM:-Qwen/Qwen3-VL-4B-Instruct}" "${LAYOUT_MODEL:-PaddlePaddle/PP-DocLayoutV3_onnx}"; do echo "[preload] downloading $m -> $HF_HOME (hf_xet)" # huggingface-cli no longer exists in huggingface_hub 2.x -> use the Python API python3 - "$m" <<'PYEOF' import sys from huggingface_hub import snapshot_download snapshot_download(sys.argv[1], ignore_patterns=["*.md", "*.gguf", "*.bin"]) PYEOF done # Cap the max input image AREA of BOTH VLMs to MAX_IMAGE_PX (default 2250000 = # 1500x1500). Notes: # - 'longest_edge' is a total pixel AREA (vLLM uses it as max_pixels), NOT an # edge length. Qwen3-VL ships 16777216 (16 Mpx = 16384 vision tokens), # GLM-OCR ships 9633792 (9.6 Mpx = ~12288 tokens). # - The V1 engine profiles the vision encoder at startup with a dummy item of # MAXIMUM feature size, so a large cap = large activation peak (this crashed # vllm-vlm on T4 with 'Available KV cache memory: -0.27 GiB'). # - Input contract of this Space: one A4 page per request, pre-resized to at # most 1200x1200 (1.44 Mpx). The 2.25 Mpx cap therefore never triggers on # real inputs; it only bounds the worst case: ~2197 vision tokens for # Qwen3-VL (1024 px/token), ~2870 for GLM-OCR (784 px/token), leaving >1000 # tokens of the 4096 context for prompt + generation. MAX_IMAGE_PX="${MAX_IMAGE_PX:-2250000}" for PC in $(find "$HF_HOME" \( -path "*GLM-OCR*" -o -path "*Qwen3-VL*" \) -name preprocessor_config.json 2>/dev/null); do python3 - "$PC" "$MAX_IMAGE_PX" <<'EOF' import json, sys p, cap = sys.argv[1], int(sys.argv[2]) d = json.load(open(p)) sz = d.get("size", {}) if "longest_edge" in sz: old = int(sz["longest_edge"]) sz["longest_edge"] = min(old, cap) d["size"] = sz json.dump(d, open(p, "w"), indent=2) print(f"[preload] capped size.longest_edge in {p}: {old} -> {sz['longest_edge']}") else: print(f"[preload] WARNING: no size.longest_edge in {p}") EOF done echo "[preload] all models cached in $HF_HOME"