Ripple-65B-Spark-Server / spark.py.diff
rlaope's picture
Ripple-65B Spark Server kit
9f69b47 verified
Raw History Blame Contribute Delete
1.98 kB
--- spark.py.orig 2026-11-05 12:40:00.417941408 +0900
+++ spark.py 2026-11-05 13:18:45.255779819 +0900
@@ -139,7 +139,7 @@
spec_args = [
"--speculative-draft-model-path", str(draft),
"--speculative-algorithm", "DFLASH", "--speculative-draft-model-quantization", "unquant",
- "--speculative-num-draft-tokens", "32", "--speculative-draft-window-size", "16384",
+ "--speculative-num-draft-tokens", os.environ.get("RIPPLE_DRAFT_TOKENS", "16"), "--speculative-draft-window-size", "16384",
"--speculative-draft-attention-backend", "triton",
] if speculative else []
args = prepare_server_args(
@@ -154,13 +154,13 @@
--attention-backend flashinfer --fp4-gemm-backend flashinfer_cutlass
--flashinfer-autotune-skip-ops fp8_gemm
--context-length 131072 --max-total-tokens 139264 --mem-fraction-static 0.78
- --max-running-requests 1 --cuda-graph-bs-decode 1
- --cuda-graph-backend-prefill disabled --disable-radix-cache
+ --max-running-requests __MRR__ --cuda-graph-bs-decode __BS__
+ --cuda-graph-backend-prefill disabled __RADIX__
--weight-loader-disable-mmap --weight-loader-drop-cache-after-load
--model-loader-extra-config '{"enable_multithread_load": false}'
--random-seed 42 --watchdog-timeout 1800 --log-level warning
--tool-call-parser ripple --reasoning-parser ripple
- """)
+ """.replace("__RADIX__", "" if os.environ.get("RIPPLE_RADIX", "1") == "1" else "--disable-radix-cache").replace("__MRR__", os.environ.get("RIPPLE_MAX_RUNNING", "3")).replace("__BS__", " ".join(str(b) for b in range(1, int(os.environ.get("RIPPLE_MAX_RUNNING", "3")) + 1))))
)
super().__init__(server_args=args)
self._chat = OpenAIServingChat(self.tokenizer_manager, self.template_manager)