File size: 798 Bytes
dfb775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | """SGLang-ROCm launcher — alternative to vLLM for serving.
Wired into `mindxtrain serve --to sglang`. SGLang ships explicit
`rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x` images. Pairs with the Mooncake
distributed-KV-cache plugin used by AMD/Xiaomi MiMo-V2.5-Pro.
"""
from __future__ import annotations
from pathlib import Path
from mindxtrain.config.schema import ServeCfg
def build_sglang_command(cfg: ServeCfg, model_dir: Path) -> list[str]:
"""Return argv for booting SGLang-ROCm against a quantized checkpoint."""
return [
"python",
"-m",
"sglang.launch_server",
"--model-path",
str(model_dir),
"--port",
str(cfg.port),
"--mem-fraction-static",
"0.85",
"--tp",
str(cfg.tensor_parallel),
]
|