Update README.md
Browse files
README.md
CHANGED
|
@@ -482,84 +482,6 @@ python -m sglang.launch_server \
|
|
| 482 |
|
| 483 |
</details>
|
| 484 |
|
| 485 |
-
<details>
|
| 486 |
-
<summary><strong>B300: Balanced (4 GPUs, TP4/EP1)</strong></summary>
|
| 487 |
-
|
| 488 |
-
#### B300: Balanced (4 GPUs, TP4/EP1)
|
| 489 |
-
|
| 490 |
-
For serving several users concurrently with MegaMoE.
|
| 491 |
-
|
| 492 |
-
```bash
|
| 493 |
-
export CUDA_VISIBLE_DEVICES=0,1,2,3
|
| 494 |
-
|
| 495 |
-
python -m sglang.launch_server \
|
| 496 |
-
--trust-remote-code \
|
| 497 |
-
--model-path stepfun-ai/Step-5-Preview \
|
| 498 |
-
--served-model-name step5 \
|
| 499 |
-
--tp 4 \
|
| 500 |
-
--ep-size 1 \
|
| 501 |
-
--moe-a2a-backend megamoe \
|
| 502 |
-
--attention-backend fa4 \
|
| 503 |
-
--page-size 64 \
|
| 504 |
-
--kv-cache-dtype bfloat16 \
|
| 505 |
-
--mem-fraction-static 0.9 \
|
| 506 |
-
--swa-full-tokens-ratio 0.03 \
|
| 507 |
-
--chunked-prefill-size 16384 \
|
| 508 |
-
--max-running-requests 64 \
|
| 509 |
-
--cuda-graph-backend-prefill breakable \
|
| 510 |
-
--flashinfer-allreduce-fusion-backend trtllm \
|
| 511 |
-
--speculative-algorithm EAGLE \
|
| 512 |
-
--speculative-num-steps 3 \
|
| 513 |
-
--speculative-eagle-topk 1 \
|
| 514 |
-
--speculative-num-draft-tokens 4 \
|
| 515 |
-
--enable-multi-layer-eagle \
|
| 516 |
-
--reasoning-parser step3p5 \
|
| 517 |
-
--tool-call-parser step3p5 \
|
| 518 |
-
--host 0.0.0.0 \
|
| 519 |
-
--port 30000
|
| 520 |
-
```
|
| 521 |
-
|
| 522 |
-
</details>
|
| 523 |
-
|
| 524 |
-
<details>
|
| 525 |
-
<summary><strong>B300: High throughput (8 GPUs, TP8/Attention DP2/EP8)</strong></summary>
|
| 526 |
-
|
| 527 |
-
#### B300: High throughput (8 GPUs, TP8/Attention DP2/EP8)
|
| 528 |
-
|
| 529 |
-
For batch generation and high-concurrency serving.
|
| 530 |
-
|
| 531 |
-
```bash
|
| 532 |
-
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 533 |
-
|
| 534 |
-
python -m sglang.launch_server \
|
| 535 |
-
--trust-remote-code \
|
| 536 |
-
--model-path stepfun-ai/Step-5-Preview \
|
| 537 |
-
--served-model-name step5 \
|
| 538 |
-
--tp 8 \
|
| 539 |
-
--attn-dp-size 2 \
|
| 540 |
-
--ep-size 8 \
|
| 541 |
-
--moe-runner-backend flashinfer_trtllm \
|
| 542 |
-
--attention-backend fa4 \
|
| 543 |
-
--page-size 64 \
|
| 544 |
-
--kv-cache-dtype bfloat16 \
|
| 545 |
-
--mem-fraction-static 0.9 \
|
| 546 |
-
--swa-full-tokens-ratio 0.03 \
|
| 547 |
-
--chunked-prefill-size 16384 \
|
| 548 |
-
--max-running-requests 64 \
|
| 549 |
-
--cuda-graph-backend-prefill disabled \
|
| 550 |
-
--flashinfer-allreduce-fusion-backend trtllm \
|
| 551 |
-
--speculative-algorithm EAGLE \
|
| 552 |
-
--speculative-num-steps 3 \
|
| 553 |
-
--speculative-eagle-topk 1 \
|
| 554 |
-
--speculative-num-draft-tokens 4 \
|
| 555 |
-
--enable-multi-layer-eagle \
|
| 556 |
-
--reasoning-parser step3p5 \
|
| 557 |
-
--tool-call-parser step3p5 \
|
| 558 |
-
--host 0.0.0.0 \
|
| 559 |
-
--port 30000
|
| 560 |
-
```
|
| 561 |
-
|
| 562 |
-
</details>
|
| 563 |
|
| 564 |
<details>
|
| 565 |
<summary><strong>H200: Balanced (8 GPUs, TP8/EP8)</strong></summary>
|
|
|
|
| 482 |
|
| 483 |
</details>
|
| 484 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 485 |
|
| 486 |
<details>
|
| 487 |
<summary><strong>H200: Balanced (8 GPUs, TP8/EP8)</strong></summary>
|