beyoru commited on
Commit
297fc05
·
verified ·
1 Parent(s): e05fa1f

Add docker-compose.vllm.yaml: sample serving config

Browse files
Files changed (1) hide show
  1. docker-compose.vllm.yaml +13 -0
docker-compose.vllm.yaml CHANGED
@@ -28,6 +28,7 @@ services:
28
  --model /models/KAT-Coder-V2.5-Dev-VL
29
  --served-model-name kat-vl
30
  --max-model-len 32768
 
31
  --gpu-memory-utilization 0.90
32
  --trust-remote-code
33
 
@@ -38,3 +39,15 @@ services:
38
  # KV cache o context day rat lon. Muon vuot native thi ap YaRN y het huong dan cua
39
  # Qwen/Qwen3.6-35B-A3B (tham so RoPE o day khong doi so voi upstream) — luu y moi
40
  # framework deu cai YaRN TINH nen bat len se lam giam chat luong prompt ngan.
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  --model /models/KAT-Coder-V2.5-Dev-VL
29
  --served-model-name kat-vl
30
  --max-model-len 32768
31
+ --max-num-batched-tokens 8192
32
  --gpu-memory-utilization 0.90
33
  --trust-remote-code
34
 
 
39
  # KV cache o context day rat lon. Muon vuot native thi ap YaRN y het huong dan cua
40
  # Qwen/Qwen3.6-35B-A3B (tham so RoPE o day khong doi so voi upstream) — luu y moi
41
  # framework deu cai YaRN TINH nen bat len se lam giam chat luong prompt ngan.
42
+
43
+ # ⚠️ `--max-num-batched-tokens 8192` la BAT BUOC, dung bo di.
44
+ #
45
+ # Kien truc nay lai attention + Gated DeltaNet. Khi prefix caching bat (mac dinh o
46
+ # vLLM moi), vLLM ep `mamba_cache_mode='align'`, roi nang attention block size len
47
+ # 2096 de "attention page size >= mamba page size". Mac dinh max_num_batched_tokens
48
+ # chi 2048 => assert vo ngay luc khoi tao KV cache:
49
+ #
50
+ # AssertionError: In Mamba cache align mode,
51
+ # block_size (2096) must be <= max_num_batched_tokens (2048)
52
+ #
53
+ # Bat ky gia tri nao >= 2096 deu qua; 8192 con loi cho prefill.