File size: 5,953 Bytes
1e251f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | #!/bin/bash
# DSV41-OFFLOAD: serve.sh (HeNryous v5 serve_example.sh + serve.sh.diff) for experiment windows. Identical settings; the
# only additions are inert unless set: SPEC_OVERRIDE / KV_BYTES_OVERRIDE / ENGRAM (the launcher's own knob),
# EXP_DOCKER_ENV appended after DOCKER_ENV, and the launcher is offload/launch/dsv41_tp2_offload.sh (EXTRA_DOCKER_ARGS).
# Example: serve DeepSeek-V4.1-Flash-1017 EXL3 with TP2 on two DGX Sparks, using the launcher of the sudoingX recipe
# (scripts/launch/dsv41_tp2.sh from github.com/sudoingX/dgx-spark-deepseek-v41). Run on the head node; the worker is
# reached over ssh (passwordless). Usage: HEAD=<head-host> WORKER=<worker-host> bash serve/serve_example.sh
# The model folder must exist on both nodes at the same path. Start it with the server stopped (see the JIT step below).
set -euo pipefail
# 0047 adaptive verify window: margin over the accepted-draft average, smallest window, threshold above which a request
# keeps the full window (two regimes), and up to how many requests a step adapts
AK_MARGIN=${AK_MARGIN:-1.0}
AK_MIN=${AK_MIN:-2}
AK_HI=${AK_HI:-1.6}
AK_MAX_REQS=${AK_MAX_REQS:-4}
RECIPE=${RECIPE:-$HOME/dgx-spark-deepseek-v41} # git clone https://github.com/sudoingX/dgx-spark-deepseek-v41
export HEAD=${HEAD:?set HEAD=<head node hostname>} WORKER=${WORKER:?set WORKER=<worker node hostname>}
export FAB_IF=${FAB_IF:-enp1s0f1np1} FAB_HCA=${FAB_HCA:-rocep1s0f1} # sparkhead/sparkworker: port f1 is the cabled one (f0 DOWN) # fabric netdev and its RoCE device (DGX Spark names)
export IMAGE=ghcr.io/henryous/deepseek-v41-1017-dgx-spark:v5
export MODEL_DIR=${MODEL_DIR:-/srv/dsv41/model} # node-local NVMe on both (head: symlink to ~/models/...; worker: real dir; ~/models on sparkworker is NFS) # incl. the two 1017 Engram shards (scripts/download_model.sh)
export SERVED=DeepSeek-V4.1-Flash-1017-EXL3-Spark-TP2
export TEXT_ONLY=0 MAX_LEN=${MAX_LEN_OVERRIDE:-327680} MAX_SEQS=4 MAX_BATCH=4096 UTIL=0.86 # MAX_LEN_OVERRIDE: model max 1,048,576
export KV_BYTES=${KV_BYTES_OVERRIDE:-4026531840} # 3.75 GiB per rank -> 989,460 tokens; UTIL then only gates the startup check
# 0040: DSpark k = 5 at 1-2 streams, k = 4 at 3-4 streams (at most 20 verify rows, kept on exl3_mrow by 0042b)
export SPEC=${SPEC_OVERRIDE-'{"method":"dspark","num_speculative_tokens":5,"num_speculative_tokens_per_batch_size":[[1,2,5],[3,4,4]]}'}
# keep the background long-prefill threshold <= MAX_BATCH / 4, otherwise background prefills can fill a whole step and a
# new foreground request waits for them (see patches/0031c). --scheduling-policy priority: vLLM's waiting queue is
# ordered by the request field `priority` (lower first); --enable-prompt-tokens-details: usage reports cached prompt tokens.
export EXTRA="--limit-mm-per-prompt '{\"image\":4}' --mm-processor-cache-gb 0 --long-prefill-token-threshold $((MAX_BATCH / 4)) \
--scheduling-policy priority --enable-prompt-tokens-details${EXP_EXTRA_ARGS:+ $EXP_EXTRA_ARGS}"
# 0031: requests with priority >= 5 count as background and prefill only every 24th step while a foreground request runs.
# NCCL over both PCIe functions of the ConnectX port (+3 % prefill); the plugin's RDMA ring stays on one device.
# 0036: split the indexer prefill rows over both ranks from 2048 rows on. EXL3_RDMA_CPU: pin the RDMA ring's polling thread
# to an efficiency core (A725: cpus 0-4, 10-14) instead of the default last X925 core: same speed, hottest sensor ~10 C cooler.
# 0045: the decode round's TP all-gathers over the RDMA ring (up to 3 MB: DSpark Markov pairs, Engram rows, logits).
# 0046 + 0042b: the LM head on exl3_mrow up to 8 rows (1 stream); the dense layers on exl3_mrow up to 32 rows, from 9 rows
# on through 0042b's ring kernel. Recipe switches: router GEMM before the shared-expert fork, shared expert inside the
# routed-expert kernel. 0048: mHC boundary in one launch (bit-identical); 0050: mHC overlap path up to 24 rows.
# 0047: adaptive verify window (needs the 0040 schedule in SPEC), values at the top.
export DOCKER_ENV="NCCL_MIN_NCHANNELS=4 NCCL_MAX_NCHANNELS=4 VLLM_DSPARK_ROUND_MS=33.26,36.33,38.91,42.23,46.03,48.15 \
VLLM_PREFILL_EVERY_N_STEPS=24 VLLM_PREFILL_THROTTLE_MIN_PRIO=5 VLLM_SPARSE_INDEXER_MAX_LOGITS_MB=128 \
NCCL_IB_HCA=rocep1s0f1,roceP2p1s0f1 EXL3_RDMA_DEVICES=rocep1s0f1 NCCL_PROTO=^LL128 NCCL_CUMEM_ENABLE=0 \
NCCL_CUMEM_HOST_ENABLE=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
VLLM_INDEXER_TP_SHARD_MIN_ROWS=2048 EXL3_RDMA_CPU=14 \
EXL3_RDMA_AG=1 EXL3_RDMA_AG_MAX_KB=3072 VLLM_EXL3_HEAD_MROW=1 EXL3_HEAD_MROW_MAX_ROWS=8 \
VLLM_EXL3_MROW_MAX_ROWS=32 EXL3_MROW32_VARIANT=ring EXL3_MROW32_MIN_ROWS=9 \
VLLM_MOE_GATE_FIRST=1 VLLM_EXL3_MOE_SMOE_SHARED=1 \
VLLM_DSV41_MHC_FUSED_POST=2 VLLM_DSV41_MHC_OVERLAP_MAX_TOKENS=24 \
VLLM_DSV41_ADAPTIVE_K=1 VLLM_DSV41_AK_MARGIN=$AK_MARGIN VLLM_DSV41_AK_MIN=$AK_MIN VLLM_DSV41_AK_HI=$AK_HI \
VLLM_DSV41_AK_MAX_REQS=$AK_MAX_REQS${EXP_DOCKER_ENV:+ $EXP_DOCKER_ENV}"
# The plugin's JIT kernels (TensorFold experts, exl3_mrow, RDMA ring, mHC boundary, router, routed-expert decode, L2
# prefetch) on both nodes before the model loads; once built for this image, this only loads them. Building them next to
# the loaded weights can run a node out of memory. EXL3_SMOE_FLAGS is the launcher's (part of the module name).
JIT="mkdir -p \$HOME/.cache/dsv41-serve && docker run --rm --gpus all -v \$HOME/.cache/dsv41-serve:/root/.cache \
-e TORCH_EXTENSIONS_DIR=/root/.cache/torch_extensions -e MAX_JOBS=4 -e EXL3_SMOE_FLAGS=-DSMOE_RUNFLAGS=1 \
--entrypoint python3 $IMAGE \
-c 'from vllm_exl3_moe import tf_experts as t, mrow, rdma, mhc, router, smoe, l2pf; t._ext(); mrow.ext(); rdma.ext(); mhc.ext(); router.ext(); smoe.ext(); l2pf.ext()'"
bash -c "$JIT" & JIT_HEAD=$!
ssh "$WORKER" "$JIT" & JIT_WORKER=$!
wait $JIT_HEAD && wait $JIT_WORKER || { echo "JIT prebuild failed" >&2; exit 1; }
SKIP_SERVE_HOLD=1 bash "$(dirname "$0")/dsv41_tp2_offload.sh" up
|