beyoru commited on
Commit
2f308ad
·
verified ·
1 Parent(s): 543e6e4

Add docker-compose.sglang-flashqla.yaml: sample serving config

Browse files
Files changed (1) hide show
  1. docker-compose.sglang-flashqla.yaml +50 -0
docker-compose.sglang-flashqla.yaml ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SGLang + FlashQLA GDN prefill — cho beyoru/KAT-Coder-V2.5-Dev-VL.
2
+ #
3
+ # ⚠️ CHUA KIEM CHUNG. Cau hinh nay dung ve kien truc nhung chua ai chay thu.
4
+ # Bao lai ket qua o tab Community neu ban chay duoc.
5
+ #
6
+ # Vi sao ap duoc: model nay co 30/40 lop `linear_attention` (Gated DeltaNet) —
7
+ # dung loai lop ma FlashQLA tang toc o prefill. Kernel TileLang cua QwenLM fuse
8
+ # GDN chunked-prefill forward, nhanh 2-3x so voi kernel Triton tren Hopper/Blackwell.
9
+ #
10
+ # ⚠️ Windowed-MTP KHONG ap duoc cho model nay: `mtp_num_hidden_layers = 0`, tuc
11
+ # checkpoint khong co MTP/NEXTN head de self-speculate. Cac bien RK_* trong image
12
+ # combo se khong lam gi. Windowed-MTP chi dung cho Qwen/Qwen3.6-35B-A3B goc (mtp=1).
13
+ services:
14
+ kat-vl-sglang:
15
+ # Anh nay = sglang 0.5.12.post1 (nvcr 26.06) + FlashQLA + Windowed-MTP.
16
+ # Chi can FlashQLA thi dung ductransa01/sglang-flashqla:cu13-20260713 (amd64)
17
+ # hoac :cu13-20260713-arm64 (Grace GH200/GB200).
18
+ image: ductransa01/sglang-flashqla:combo-2606
19
+ container_name: kat-vl-sglang
20
+ ipc: host
21
+ ports:
22
+ - "30000:30000"
23
+ volumes:
24
+ - /path/to/models:/models
25
+ - ${HOME}/.cache/huggingface:/root/.cache/huggingface
26
+ deploy:
27
+ resources:
28
+ reservations:
29
+ devices:
30
+ - driver: nvidia
31
+ device_ids: ["0"]
32
+ capabilities: [gpu]
33
+ command: >
34
+ python3 -m sglang.launch_server
35
+ --model-path /models/KAT-Coder-V2.5-Dev-VL
36
+ --served-model-name kat-vl
37
+ --host 0.0.0.0 --port 30000
38
+ --context-length 32768
39
+ --mem-fraction-static 0.85
40
+ --trust-remote-code
41
+ --linear-attn-prefill-backend flashqla
42
+
43
+ # `--linear-attn-prefill-backend flashqla` la CO DUY NHAT can them. Khong truyen thi
44
+ # duong mac dinh (triton) khong doi mot chut nao.
45
+ #
46
+ # ⛔ DUNG truyen `--linear-attn-decode-backend flashqla` — FlashQLA khong co kernel
47
+ # decode, server se raise ValueError. Decode tu o lai triton.
48
+ #
49
+ # ⛔ DUNG dung `--language-model-only` (hoac co tuong duong cua phien ban ban dung):
50
+ # checkpoint nay CO vision tower, co do se tat no di.