| # GLM-5.3 FULL EXL3 3bpw mcg — 4x DGX Spark TP=4 (+DCP=4 for 1M) — DRAFT for ~/glm53-mia/.env.tp4 on .1 | |
| # ranks: .1 head, .2 / .3 / .5 workers (all on the 10.100.10.x CX7 fabric, same NIC names) | |
| HEAD_IP=10.100.10.1 | |
| WORKER_IP=10.100.10.2 | |
| WORKER2_IP=10.100.10.3 | |
| WORKER3_IP=10.100.10.5 | |
| WORKER_USER=keyspark | |
| NCCL_CROSS_NIC=0 | |
| HEAD_CX7_IF=enp1s0f1np1 | |
| HEAD_CX7_IB=rocep1s0f1 | |
| WORKER_CX7_IF=enP2p1s0f1np1 | |
| WORKER_CX7_IB=roceP2p1s0f1 | |
| WORKER2_CX7_IF=enp1s0f1np1 | |
| WORKER2_CX7_IB=rocep1s0f1 | |
| WORKER3_CX7_IF=enp1s0f1np1 | |
| WORKER3_CX7_IB=rocep1s0f1 | |
| NCCL_IB_GID_INDEX=3 | |
| NCCL_DEBUG=WARN | |
| TP=4 | |
| NNODES=4 | |
| PORT=8888 | |
| MASTER_PORT=29521 | |
| CONTAINER_HEAD=glm53-exl3-tp4-head | |
| CONTAINER_WORKER=glm53-exl3-tp4-w1 | |
| CONTAINER_WORKER2=glm53-exl3-tp4-w2 | |
| CONTAINER_WORKER3=glm53-exl3-tp4-w3 | |
| # model: quantized on .3 -> exported over NFS as /mnt/models-node2/<name> on every node | |
| LOCAL_MODEL_DIR=/mnt/models-node2/GLM-5.3-EXL3-3bpw | |
| MODEL=keys/GLM-5.3-EXL3-3bpw | |
| SERVED_MODEL_NAME=GLM-5.3-EXL3 | |
| IMAGE=glm53-exl3-tp4:keys3 | |
| QUANTIZATION=exl3 | |
| KV_CACHE_DTYPE=fp8 | |
| MAX_MODEL_LEN=1000000 | |
| GPU_MEM_UTIL=0.82 | |
| MAX_NUM_SEQS=8 | |
| MAX_NUM_BATCHED_TOKENS=4096 | |
| SPEC_METHOD=mtp | |
| LANGUAGE_MODEL_ONLY=1 | |
| ENFORCE_EAGER=0 | |
| EXTRA_ARGS="--decode-context-parallel-size 4 --kv-cache-memory-bytes 14092861440" | |
| LOCAL_DFLASH_DIR=/mnt/models-node2/GLM-5.3-DFlash2 | |
| DFLASH_TOKENS=7 | |
| GLM53_MIXED_PREFILL_CHUNK=2048 | |
| MTP_TOKENS=3 | |