File size: 2,850 Bytes
b4973b5
 
 
 
 
 
 
 
 
 
 
76c2cef
 
 
 
b4973b5
 
 
 
 
a0f3454
 
 
 
 
 
b4973b5
b599c96
aef2876
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b599c96
aef2876
b599c96
909e8cf
 
 
aef2876
909e8cf
b599c96
909e8cf
 
 
b599c96
aef2876
b599c96
b4973b5
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
#!/bin/bash
# Preload the 3 models from the HF Hub into LOCAL disk (HF_HOME=/tmp/hf) using
# hf_xet (chunked, parallel, fast) BEFORE starting vLLM.
# Why: reading weights through the FUSE-mounted bucket (/data) is ~12 Mo/s
# (checkpoint loads took >10 min); a direct HF Hub download is ~100x faster,
# and loading from local disk afterwards is instant (page cache).
# /tmp is ephemeral, so this re-downloads at each Space restart (~1-3 min),
# which is still far better than FUSE. The /data bucket stays mounted for
# other persistent data.
set -euo pipefail

# hf_xet is the default transfer backend in huggingface_hub 2.x; this enables its
# high-performance mode (chunked parallel downloads). HF_HUB_ENABLE_HF_TRANSFER is
# deprecated in hub 2.x.
export HF_XET_HIGH_PERFORMANCE=1
HF_HOME="${HF_HOME:-/tmp/hf}"
mkdir -p "$HF_HOME"

for m in "${MODEL_OCR:-zai-org/GLM-OCR}" "${MODEL_VLM:-Qwen/Qwen3-VL-4B-Instruct}" "${LAYOUT_MODEL:-PaddlePaddle/PP-DocLayoutV3_onnx}"; do
    echo "[preload] downloading $m -> $HF_HOME (hf_xet)"
    # huggingface-cli no longer exists in huggingface_hub 2.x -> use the Python API
    python3 - "$m" <<'PYEOF'
import sys
from huggingface_hub import snapshot_download
snapshot_download(sys.argv[1], ignore_patterns=["*.md", "*.gguf", "*.bin"])
PYEOF
done

# Cap the max input image AREA of BOTH VLMs to MAX_IMAGE_PX (default 2250000 =
# 1500x1500). Notes:
# - 'longest_edge' is a total pixel AREA (vLLM uses it as max_pixels), NOT an
#   edge length. Qwen3-VL ships 16777216 (16 Mpx = 16384 vision tokens),
#   GLM-OCR ships 9633792 (9.6 Mpx = ~12288 tokens).
# - The V1 engine profiles the vision encoder at startup with a dummy item of
#   MAXIMUM feature size, so a large cap = large activation peak (this crashed
#   vllm-vlm on T4 with 'Available KV cache memory: -0.27 GiB').
# - Input contract of this Space: one A4 page per request, pre-resized to at
#   most 1200x1200 (1.44 Mpx). The 2.25 Mpx cap therefore never triggers on
#   real inputs; it only bounds the worst case: ~2197 vision tokens for
#   Qwen3-VL (1024 px/token), ~2870 for GLM-OCR (784 px/token), leaving >1000
#   tokens of the 4096 context for prompt + generation.
MAX_IMAGE_PX="${MAX_IMAGE_PX:-2250000}"
for PC in $(find "$HF_HOME" \( -path "*GLM-OCR*" -o -path "*Qwen3-VL*" \) -name preprocessor_config.json 2>/dev/null); do
    python3 - "$PC" "$MAX_IMAGE_PX" <<'EOF'
import json, sys
p, cap = sys.argv[1], int(sys.argv[2])
d = json.load(open(p))
sz = d.get("size", {})
if "longest_edge" in sz:
    old = int(sz["longest_edge"])
    sz["longest_edge"] = min(old, cap)
    d["size"] = sz
    json.dump(d, open(p, "w"), indent=2)
    print(f"[preload] capped size.longest_edge in {p}: {old} -> {sz['longest_edge']}")
else:
    print(f"[preload] WARNING: no size.longest_edge in {p}")
EOF
done

echo "[preload] all models cached in $HF_HOME"