File size: 1,349 Bytes
c226d57 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | # GLM-5.3 FULL EXL3 3bpw mcg — 4x DGX Spark TP=4 (+DCP=4 for 1M) — DRAFT for ~/glm53-mia/.env.tp4 on .1
# ranks: .1 head, .2 / .3 / .5 workers (all on the 10.100.10.x CX7 fabric, same NIC names)
HEAD_IP=10.100.10.1
WORKER_IP=10.100.10.2
WORKER2_IP=10.100.10.3
WORKER3_IP=10.100.10.5
WORKER_USER=keyspark
NCCL_CROSS_NIC=0
HEAD_CX7_IF=enp1s0f1np1
HEAD_CX7_IB=rocep1s0f1
WORKER_CX7_IF=enP2p1s0f1np1
WORKER_CX7_IB=roceP2p1s0f1
WORKER2_CX7_IF=enp1s0f1np1
WORKER2_CX7_IB=rocep1s0f1
WORKER3_CX7_IF=enp1s0f1np1
WORKER3_CX7_IB=rocep1s0f1
NCCL_IB_GID_INDEX=3
NCCL_DEBUG=WARN
TP=4
NNODES=4
PORT=8888
MASTER_PORT=29521
CONTAINER_HEAD=glm53-exl3-tp4-head
CONTAINER_WORKER=glm53-exl3-tp4-w1
CONTAINER_WORKER2=glm53-exl3-tp4-w2
CONTAINER_WORKER3=glm53-exl3-tp4-w3
# model: quantized on .3 -> exported over NFS as /mnt/models-node2/<name> on every node
LOCAL_MODEL_DIR=/mnt/models-node2/GLM-5.3-EXL3-3bpw
MODEL=keys/GLM-5.3-EXL3-3bpw
SERVED_MODEL_NAME=GLM-5.3-EXL3
IMAGE=glm53-exl3-tp4:keys3
QUANTIZATION=exl3
KV_CACHE_DTYPE=fp8
MAX_MODEL_LEN=1000000
GPU_MEM_UTIL=0.82
MAX_NUM_SEQS=8
MAX_NUM_BATCHED_TOKENS=4096
SPEC_METHOD=mtp
LANGUAGE_MODEL_ONLY=1
ENFORCE_EAGER=0
EXTRA_ARGS="--decode-context-parallel-size 4 --kv-cache-memory-bytes 14092861440"
LOCAL_DFLASH_DIR=/mnt/models-node2/GLM-5.3-DFlash2
DFLASH_TOKENS=7
GLM53_MIXED_PREFILL_CHUNK=2048
MTP_TOKENS=3
|