granularity results tierb_study_20260826_144644 @ 2026-08-26T17:19:17Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
Browse files- .gitattributes +1 -0
- tierb_study_20260826_144644/echo/tierbprobe/manifest.txt +2 -2
- tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +42 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv +40 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/run.env +61 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log +13 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +67 -67
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +0 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/mem_samples.csv +256 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/run.env +61 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/train.log +0 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/mem_samples.csv +232 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/run.env +61 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/train.log +0 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +62 -62
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +0 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/mem_samples.csv +272 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/run.env +61 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/train.log +3 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv +137 -145
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log +2 -2
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv +109 -109
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log +2 -2
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +7 -2
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +2 -0
- tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv +7 -2
- tierb_study_20260826_144644/hf_watcher.log +3 -0
- tierb_study_20260826_144644/study.log +44 -0
.gitattributes
CHANGED
|
@@ -37,3 +37,4 @@ tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__mat
|
|
| 37 |
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
| 38 |
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
| 39 |
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 37 |
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
| 38 |
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
| 39 |
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
tierb_study_20260826_144644/echo/tierbprobe/manifest.txt
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
-
timestamp: 2026-08-26T17:
|
| 2 |
host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
|
| 3 |
gpu: NVIDIA B200 x8
|
| 4 |
driver: 580.95.05
|
| 5 |
study_scripts_branch: granularity-echo
|
| 6 |
-
study_scripts_commit:
|
| 7 |
campaign: echo
|
| 8 |
backend: echo
|
| 9 |
phase: tierbprobe
|
|
|
|
| 1 |
+
timestamp: 2026-08-26T17:09:26+00:00
|
| 2 |
host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
|
| 3 |
gpu: NVIDIA B200 x8
|
| 4 |
driver: 580.95.05
|
| 5 |
study_scripts_branch: granularity-echo
|
| 6 |
+
study_scripts_commit: 6553011
|
| 7 |
campaign: echo
|
| 8 |
backend: echo
|
| 9 |
phase: tierbprobe
|
tierb_study_20260826_144644/echo/tierbprobe/provenance.txt
CHANGED
|
@@ -46,3 +46,45 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_
|
|
| 46 |
---- toolchain -----------------------------------------------
|
| 47 |
nvcc : release 12.9, V12.9.86
|
| 48 |
gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
---- toolchain -----------------------------------------------
|
| 47 |
nvcc : release 12.9, V12.9.86
|
| 48 |
gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
|
| 49 |
+
python : 3.12.3
|
| 50 |
+
torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
|
| 51 |
+
torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
|
| 52 |
+
transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
|
| 53 |
+
deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
|
| 54 |
+
ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
|
| 55 |
+
moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
|
| 56 |
+
megatron : 0.14.0rc5 [None]
|
| 57 |
+
flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
|
| 58 |
+
numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
|
| 59 |
+
huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
|
| 60 |
+
|
| 61 |
+
---- repositories --------------------------------------------
|
| 62 |
+
Megatron-MoE-Echo token-drop-analysis b099fd8494b0
|
| 63 |
+
Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21
|
| 64 |
+
UltraEP main 94cab099b44f
|
| 65 |
+
DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
|
| 66 |
+
MoonEP megatron-integration c27d35da15f2
|
| 67 |
+
blackwell_run_scripts granularity-echo 655301106287
|
| 68 |
+
|
| 69 |
+
---- environment ---------------------------------------------
|
| 70 |
+
CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
|
| 71 |
+
CUDA_DRIVER_VERSION=575.57.08
|
| 72 |
+
CUDA_HOME=/usr/local/cuda
|
| 73 |
+
CUDA_MODULE_LOADING=LAZY
|
| 74 |
+
CUDA_VERSION=12.9.1.010
|
| 75 |
+
HF_REMOTE_ROOT=tierb_study_20260826_144644
|
| 76 |
+
NCCL_VERSION=2.27.3
|
| 77 |
+
NVSHMEM_VERSION=3.2.5
|
| 78 |
+
PYTORCH_BUILD_NUMBER=0
|
| 79 |
+
PYTORCH_BUILD_VERSION=2.8.0a0+5228986
|
| 80 |
+
PYTORCH_HOME=/opt/pytorch/pytorch
|
| 81 |
+
PYTORCH_VERSION=2.8.0a0+5228986
|
| 82 |
+
TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
|
| 83 |
+
TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
|
| 84 |
+
TORCH_NCCL_USE_COMM_NONBLOCKING=0
|
| 85 |
+
|
| 86 |
+
---- shared library resolution -------------------------------
|
| 87 |
+
libnvshmem_host.so: not in ldconfig
|
| 88 |
+
libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
|
| 89 |
+
lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
|
| 90 |
+
-rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 4
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 1
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=DSV2_LITE_L24
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=token-drop-analysis
|
| 6 |
+
commit=b099fd84
|
| 7 |
+
num_layers=24
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=10944
|
| 10 |
+
moe_ffn_hidden=1408
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=6
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=8650752
|
| 15 |
+
heads=16
|
| 16 |
+
kv_groups=0
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=2816
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=1
|
| 22 |
+
dense_first_layers=1
|
| 23 |
+
moe_layers=23
|
| 24 |
+
rotary_base=10000
|
| 25 |
+
rotary_scaling=40
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=seq_aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=49152
|
| 39 |
+
mean_tokens_per_expert=6144
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.0234375
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[17:19:04] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe)
|
| 2 |
+
==============================================================
|
| 3 |
+
DSV2_LITE_L24 / echo (slots/rank=8 policy=matched)
|
| 4 |
+
E=64 K=6 H'=1408 m=8
|
| 5 |
+
L=24 seq=8192 gbs=8 (1 microbatches)
|
| 6 |
+
cap_mode=static cap=1.0234375 cg=0 recompute=none
|
| 7 |
+
memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/memory_snapshots/snapshot.pickle
|
| 8 |
+
repo=/workspace/Megatron-MoE-Echo @ b099fd84
|
| 9 |
+
==============================================================
|
| 10 |
+
W0826 17:19:06.243000 350624 torch/distributed/run.py:766]
|
| 11 |
+
W0826 17:19:06.243000 350624 torch/distributed/run.py:766] *****************************************
|
| 12 |
+
W0826 17:19:06.243000 350624 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 13 |
+
W0826 17:19:06.243000 350624 torch/distributed/run.py:766] *****************************************
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv
CHANGED
|
@@ -14,7 +14,7 @@
|
|
| 14 |
5, 0
|
| 15 |
6, 0
|
| 16 |
7, 0
|
| 17 |
-
0,
|
| 18 |
1, 1
|
| 19 |
2, 1
|
| 20 |
3, 1
|
|
@@ -43,17 +43,17 @@
|
|
| 43 |
2, 629
|
| 44 |
3, 629
|
| 45 |
4, 629
|
| 46 |
-
5,
|
| 47 |
-
6,
|
| 48 |
-
7,
|
| 49 |
-
0,
|
| 50 |
-
1,
|
| 51 |
-
2,
|
| 52 |
-
3,
|
| 53 |
-
4,
|
| 54 |
-
5,
|
| 55 |
-
6,
|
| 56 |
-
7,
|
| 57 |
0, 1955
|
| 58 |
1, 1955
|
| 59 |
2, 1955
|
|
@@ -78,14 +78,14 @@
|
|
| 78 |
5, 32075
|
| 79 |
6, 32209
|
| 80 |
7, 32229
|
| 81 |
-
0,
|
| 82 |
-
1,
|
| 83 |
-
2,
|
| 84 |
-
3,
|
| 85 |
-
4,
|
| 86 |
-
5,
|
| 87 |
-
6,
|
| 88 |
-
7,
|
| 89 |
0, 37783
|
| 90 |
1, 37867
|
| 91 |
2, 37717
|
|
@@ -102,14 +102,14 @@
|
|
| 102 |
5, 37801
|
| 103 |
6, 37937
|
| 104 |
7, 37955
|
| 105 |
-
0,
|
| 106 |
-
1,
|
| 107 |
-
2,
|
| 108 |
-
3,
|
| 109 |
-
4,
|
| 110 |
-
5,
|
| 111 |
-
6,
|
| 112 |
-
7,
|
| 113 |
0, 38819
|
| 114 |
1, 38903
|
| 115 |
2, 38753
|
|
@@ -142,13 +142,13 @@
|
|
| 142 |
5, 38759
|
| 143 |
6, 38893
|
| 144 |
7, 38913
|
| 145 |
-
0,
|
| 146 |
-
1,
|
| 147 |
-
2,
|
| 148 |
-
3,
|
| 149 |
-
4,
|
| 150 |
-
5,
|
| 151 |
-
6,
|
| 152 |
7, 121615
|
| 153 |
0, 121857
|
| 154 |
1, 122153
|
|
@@ -182,14 +182,14 @@
|
|
| 182 |
5, 122063
|
| 183 |
6, 122133
|
| 184 |
7, 122153
|
| 185 |
-
0,
|
| 186 |
-
1,
|
| 187 |
-
2,
|
| 188 |
-
3,
|
| 189 |
-
4,
|
| 190 |
-
5,
|
| 191 |
-
6,
|
| 192 |
-
7,
|
| 193 |
0, 123719
|
| 194 |
1, 126063
|
| 195 |
2, 125893
|
|
@@ -198,21 +198,21 @@
|
|
| 198 |
5, 125973
|
| 199 |
6, 126043
|
| 200 |
7, 126063
|
| 201 |
-
0,
|
| 202 |
-
1,
|
| 203 |
-
2,
|
| 204 |
-
3,
|
| 205 |
-
4,
|
| 206 |
-
5,
|
| 207 |
-
6,
|
| 208 |
-
7,
|
| 209 |
0, 139007
|
| 210 |
1, 138657
|
| 211 |
2, 138497
|
| 212 |
3, 138493
|
| 213 |
4, 138493
|
| 214 |
5, 138567
|
| 215 |
-
6,
|
| 216 |
7, 140961
|
| 217 |
0, 139007
|
| 218 |
1, 138657
|
|
@@ -220,7 +220,7 @@
|
|
| 220 |
3, 138493
|
| 221 |
4, 138493
|
| 222 |
5, 138567
|
| 223 |
-
6,
|
| 224 |
7, 140961
|
| 225 |
0, 139007
|
| 226 |
1, 138657
|
|
@@ -228,7 +228,7 @@
|
|
| 228 |
3, 138493
|
| 229 |
4, 138493
|
| 230 |
5, 138567
|
| 231 |
-
6,
|
| 232 |
7, 140961
|
| 233 |
0, 139007
|
| 234 |
1, 138657
|
|
@@ -236,21 +236,21 @@
|
|
| 236 |
3, 138493
|
| 237 |
4, 138493
|
| 238 |
5, 138567
|
| 239 |
-
6,
|
| 240 |
7, 140961
|
| 241 |
-
0,
|
| 242 |
-
1,
|
| 243 |
-
2,
|
| 244 |
-
3,
|
| 245 |
-
4,
|
| 246 |
-
5,
|
| 247 |
-
6,
|
| 248 |
-
7,
|
| 249 |
-
0,
|
| 250 |
1, 778
|
| 251 |
2, 778
|
| 252 |
3, 778
|
| 253 |
4, 778
|
| 254 |
-
5,
|
| 255 |
-
6,
|
| 256 |
-
7,
|
|
|
|
| 14 |
5, 0
|
| 15 |
6, 0
|
| 16 |
7, 0
|
| 17 |
+
0, 2
|
| 18 |
1, 1
|
| 19 |
2, 1
|
| 20 |
3, 1
|
|
|
|
| 43 |
2, 629
|
| 44 |
3, 629
|
| 45 |
4, 629
|
| 46 |
+
5, 165
|
| 47 |
+
6, 629
|
| 48 |
+
7, 629
|
| 49 |
+
0, 895
|
| 50 |
+
1, 903
|
| 51 |
+
2, 909
|
| 52 |
+
3, 895
|
| 53 |
+
4, 893
|
| 54 |
+
5, 897
|
| 55 |
+
6, 893
|
| 56 |
+
7, 911
|
| 57 |
0, 1955
|
| 58 |
1, 1955
|
| 59 |
2, 1955
|
|
|
|
| 78 |
5, 32075
|
| 79 |
6, 32209
|
| 80 |
7, 32229
|
| 81 |
+
0, 36383
|
| 82 |
+
1, 36469
|
| 83 |
+
2, 36335
|
| 84 |
+
3, 36331
|
| 85 |
+
4, 36321
|
| 86 |
+
5, 36305
|
| 87 |
+
6, 36427
|
| 88 |
+
7, 36471
|
| 89 |
0, 37783
|
| 90 |
1, 37867
|
| 91 |
2, 37717
|
|
|
|
| 102 |
5, 37801
|
| 103 |
6, 37937
|
| 104 |
7, 37955
|
| 105 |
+
0, 38055
|
| 106 |
+
1, 38143
|
| 107 |
+
2, 37995
|
| 108 |
+
3, 37977
|
| 109 |
+
4, 37977
|
| 110 |
+
5, 37963
|
| 111 |
+
6, 38137
|
| 112 |
+
7, 38137
|
| 113 |
0, 38819
|
| 114 |
1, 38903
|
| 115 |
2, 38753
|
|
|
|
| 142 |
5, 38759
|
| 143 |
6, 38893
|
| 144 |
7, 38913
|
| 145 |
+
0, 121321
|
| 146 |
+
1, 121611
|
| 147 |
+
2, 121443
|
| 148 |
+
3, 121449
|
| 149 |
+
4, 121437
|
| 150 |
+
5, 121529
|
| 151 |
+
6, 121599
|
| 152 |
7, 121615
|
| 153 |
0, 121857
|
| 154 |
1, 122153
|
|
|
|
| 182 |
5, 122063
|
| 183 |
6, 122133
|
| 184 |
7, 122153
|
| 185 |
+
0, 122469
|
| 186 |
+
1, 122787
|
| 187 |
+
2, 122655
|
| 188 |
+
3, 122611
|
| 189 |
+
4, 122611
|
| 190 |
+
5, 122655
|
| 191 |
+
6, 122755
|
| 192 |
+
7, 122735
|
| 193 |
0, 123719
|
| 194 |
1, 126063
|
| 195 |
2, 125893
|
|
|
|
| 198 |
5, 125973
|
| 199 |
6, 126043
|
| 200 |
7, 126063
|
| 201 |
+
0, 124319
|
| 202 |
+
1, 126645
|
| 203 |
+
2, 126519
|
| 204 |
+
3, 126505
|
| 205 |
+
4, 126489
|
| 206 |
+
5, 126613
|
| 207 |
+
6, 126663
|
| 208 |
+
7, 126703
|
| 209 |
0, 139007
|
| 210 |
1, 138657
|
| 211 |
2, 138497
|
| 212 |
3, 138493
|
| 213 |
4, 138493
|
| 214 |
5, 138567
|
| 215 |
+
6, 140755
|
| 216 |
7, 140961
|
| 217 |
0, 139007
|
| 218 |
1, 138657
|
|
|
|
| 220 |
3, 138493
|
| 221 |
4, 138493
|
| 222 |
5, 138567
|
| 223 |
+
6, 140755
|
| 224 |
7, 140961
|
| 225 |
0, 139007
|
| 226 |
1, 138657
|
|
|
|
| 228 |
3, 138493
|
| 229 |
4, 138493
|
| 230 |
5, 138567
|
| 231 |
+
6, 140755
|
| 232 |
7, 140961
|
| 233 |
0, 139007
|
| 234 |
1, 138657
|
|
|
|
| 236 |
3, 138493
|
| 237 |
4, 138493
|
| 238 |
5, 138567
|
| 239 |
+
6, 140755
|
| 240 |
7, 140961
|
| 241 |
+
0, 139007
|
| 242 |
+
1, 138657
|
| 243 |
+
2, 138497
|
| 244 |
+
3, 138493
|
| 245 |
+
4, 138493
|
| 246 |
+
5, 138567
|
| 247 |
+
6, 140755
|
| 248 |
+
7, 140961
|
| 249 |
+
0, 2057
|
| 250 |
1, 778
|
| 251 |
2, 778
|
| 252 |
3, 778
|
| 253 |
4, 778
|
| 254 |
+
5, 778
|
| 255 |
+
6, 1034
|
| 256 |
+
7, 140013
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 4
|
| 43 |
+
2, 4
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1975
|
| 58 |
+
1, 1975
|
| 59 |
+
2, 1975
|
| 60 |
+
3, 1975
|
| 61 |
+
4, 1975
|
| 62 |
+
5, 1975
|
| 63 |
+
6, 1975
|
| 64 |
+
7, 1975
|
| 65 |
+
0, 47803
|
| 66 |
+
1, 47803
|
| 67 |
+
2, 47803
|
| 68 |
+
3, 47803
|
| 69 |
+
4, 47515
|
| 70 |
+
5, 47803
|
| 71 |
+
6, 47803
|
| 72 |
+
7, 47803
|
| 73 |
+
0, 49279
|
| 74 |
+
1, 49279
|
| 75 |
+
2, 49279
|
| 76 |
+
3, 49279
|
| 77 |
+
4, 48991
|
| 78 |
+
5, 49279
|
| 79 |
+
6, 49279
|
| 80 |
+
7, 49279
|
| 81 |
+
0, 49929
|
| 82 |
+
1, 49929
|
| 83 |
+
2, 49929
|
| 84 |
+
3, 49929
|
| 85 |
+
4, 49641
|
| 86 |
+
5, 49927
|
| 87 |
+
6, 49929
|
| 88 |
+
7, 49929
|
| 89 |
+
0, 53697
|
| 90 |
+
1, 53697
|
| 91 |
+
2, 53697
|
| 92 |
+
3, 53697
|
| 93 |
+
4, 53409
|
| 94 |
+
5, 53697
|
| 95 |
+
6, 53697
|
| 96 |
+
7, 53697
|
| 97 |
+
0, 54105
|
| 98 |
+
1, 54105
|
| 99 |
+
2, 54105
|
| 100 |
+
3, 54105
|
| 101 |
+
4, 53817
|
| 102 |
+
5, 54105
|
| 103 |
+
6, 54105
|
| 104 |
+
7, 54105
|
| 105 |
+
0, 54555
|
| 106 |
+
1, 54553
|
| 107 |
+
2, 54555
|
| 108 |
+
3, 54555
|
| 109 |
+
4, 54267
|
| 110 |
+
5, 54555
|
| 111 |
+
6, 54555
|
| 112 |
+
7, 54527
|
| 113 |
+
0, 55543
|
| 114 |
+
1, 55543
|
| 115 |
+
2, 55543
|
| 116 |
+
3, 55543
|
| 117 |
+
4, 55255
|
| 118 |
+
5, 55543
|
| 119 |
+
6, 55543
|
| 120 |
+
7, 55543
|
| 121 |
+
0, 56071
|
| 122 |
+
1, 56071
|
| 123 |
+
2, 56071
|
| 124 |
+
3, 56071
|
| 125 |
+
4, 55783
|
| 126 |
+
5, 56071
|
| 127 |
+
6, 56071
|
| 128 |
+
7, 56071
|
| 129 |
+
0, 57013
|
| 130 |
+
1, 57013
|
| 131 |
+
2, 57013
|
| 132 |
+
3, 57013
|
| 133 |
+
4, 56725
|
| 134 |
+
5, 57013
|
| 135 |
+
6, 57013
|
| 136 |
+
7, 57013
|
| 137 |
+
0, 57655
|
| 138 |
+
1, 57655
|
| 139 |
+
2, 57655
|
| 140 |
+
3, 57655
|
| 141 |
+
4, 57367
|
| 142 |
+
5, 57655
|
| 143 |
+
6, 57655
|
| 144 |
+
7, 57655
|
| 145 |
+
0, 96515
|
| 146 |
+
1, 96507
|
| 147 |
+
2, 96507
|
| 148 |
+
3, 96507
|
| 149 |
+
4, 96531
|
| 150 |
+
5, 96507
|
| 151 |
+
6, 96507
|
| 152 |
+
7, 96507
|
| 153 |
+
0, 97161
|
| 154 |
+
1, 97153
|
| 155 |
+
2, 97153
|
| 156 |
+
3, 97153
|
| 157 |
+
4, 97177
|
| 158 |
+
5, 97153
|
| 159 |
+
6, 97153
|
| 160 |
+
7, 97153
|
| 161 |
+
0, 97163
|
| 162 |
+
1, 97155
|
| 163 |
+
2, 97155
|
| 164 |
+
3, 97155
|
| 165 |
+
4, 97179
|
| 166 |
+
5, 97155
|
| 167 |
+
6, 97155
|
| 168 |
+
7, 97155
|
| 169 |
+
0, 97163
|
| 170 |
+
1, 97155
|
| 171 |
+
2, 97155
|
| 172 |
+
3, 97155
|
| 173 |
+
4, 97179
|
| 174 |
+
5, 97155
|
| 175 |
+
6, 97155
|
| 176 |
+
7, 97155
|
| 177 |
+
0, 97163
|
| 178 |
+
1, 97155
|
| 179 |
+
2, 97155
|
| 180 |
+
3, 97155
|
| 181 |
+
4, 97179
|
| 182 |
+
5, 97155
|
| 183 |
+
6, 97155
|
| 184 |
+
7, 97155
|
| 185 |
+
0, 97615
|
| 186 |
+
1, 97561
|
| 187 |
+
2, 97597
|
| 188 |
+
3, 97551
|
| 189 |
+
4, 97567
|
| 190 |
+
5, 97529
|
| 191 |
+
6, 97535
|
| 192 |
+
7, 97529
|
| 193 |
+
0, 102247
|
| 194 |
+
1, 101855
|
| 195 |
+
2, 101855
|
| 196 |
+
3, 101855
|
| 197 |
+
4, 101879
|
| 198 |
+
5, 102239
|
| 199 |
+
6, 101855
|
| 200 |
+
7, 102239
|
| 201 |
+
0, 108141
|
| 202 |
+
1, 107647
|
| 203 |
+
2, 107639
|
| 204 |
+
3, 107661
|
| 205 |
+
4, 108051
|
| 206 |
+
5, 108085
|
| 207 |
+
6, 107609
|
| 208 |
+
7, 108057
|
| 209 |
+
0, 132791
|
| 210 |
+
1, 132543
|
| 211 |
+
2, 132543
|
| 212 |
+
3, 132543
|
| 213 |
+
4, 132579
|
| 214 |
+
5, 132927
|
| 215 |
+
6, 132543
|
| 216 |
+
7, 132927
|
| 217 |
+
0, 132791
|
| 218 |
+
1, 132543
|
| 219 |
+
2, 132543
|
| 220 |
+
3, 132543
|
| 221 |
+
4, 132579
|
| 222 |
+
5, 132927
|
| 223 |
+
6, 132543
|
| 224 |
+
7, 132927
|
| 225 |
+
0, 132791
|
| 226 |
+
1, 132543
|
| 227 |
+
2, 132543
|
| 228 |
+
3, 132543
|
| 229 |
+
4, 132559
|
| 230 |
+
5, 132865
|
| 231 |
+
6, 132449
|
| 232 |
+
7, 132893
|
| 233 |
+
0, 3366
|
| 234 |
+
1, 3622
|
| 235 |
+
2, 131295
|
| 236 |
+
3, 131295
|
| 237 |
+
4, 4646
|
| 238 |
+
5, 3366
|
| 239 |
+
6, 3366
|
| 240 |
+
7, 3366
|
| 241 |
+
0, 0
|
| 242 |
+
1, 0
|
| 243 |
+
2, 0
|
| 244 |
+
3, 0
|
| 245 |
+
4, 0
|
| 246 |
+
5, 0
|
| 247 |
+
6, 0
|
| 248 |
+
7, 0
|
| 249 |
+
0, 0
|
| 250 |
+
1, 0
|
| 251 |
+
2, 0
|
| 252 |
+
3, 0
|
| 253 |
+
4, 0
|
| 254 |
+
5, 0
|
| 255 |
+
6, 0
|
| 256 |
+
7, 0
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=QWEN3_235B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=token-drop-analysis
|
| 6 |
+
commit=b099fd84
|
| 7 |
+
num_layers=12
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=12288
|
| 10 |
+
moe_ffn_hidden=1536
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=18874368
|
| 15 |
+
heads=64
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=12
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=0
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=65536
|
| 39 |
+
mean_tokens_per_expert=4096
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=16
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.03125
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 4
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 821
|
| 42 |
+
1, 819
|
| 43 |
+
2, 821
|
| 44 |
+
3, 819
|
| 45 |
+
4, 819
|
| 46 |
+
5, 821
|
| 47 |
+
6, 819
|
| 48 |
+
7, 819
|
| 49 |
+
0, 1537
|
| 50 |
+
1, 1537
|
| 51 |
+
2, 1537
|
| 52 |
+
3, 1537
|
| 53 |
+
4, 1537
|
| 54 |
+
5, 1537
|
| 55 |
+
6, 1537
|
| 56 |
+
7, 1537
|
| 57 |
+
0, 2551
|
| 58 |
+
1, 2551
|
| 59 |
+
2, 2551
|
| 60 |
+
3, 2551
|
| 61 |
+
4, 2551
|
| 62 |
+
5, 2551
|
| 63 |
+
6, 2551
|
| 64 |
+
7, 2551
|
| 65 |
+
0, 48097
|
| 66 |
+
1, 48085
|
| 67 |
+
2, 48091
|
| 68 |
+
3, 48087
|
| 69 |
+
4, 47783
|
| 70 |
+
5, 48119
|
| 71 |
+
6, 48067
|
| 72 |
+
7, 48125
|
| 73 |
+
0, 49643
|
| 74 |
+
1, 49653
|
| 75 |
+
2, 49703
|
| 76 |
+
3, 49683
|
| 77 |
+
4, 49425
|
| 78 |
+
5, 49713
|
| 79 |
+
6, 49703
|
| 80 |
+
7, 49693
|
| 81 |
+
0, 53697
|
| 82 |
+
1, 53697
|
| 83 |
+
2, 53697
|
| 84 |
+
3, 53697
|
| 85 |
+
4, 53409
|
| 86 |
+
5, 53697
|
| 87 |
+
6, 53697
|
| 88 |
+
7, 53697
|
| 89 |
+
0, 54105
|
| 90 |
+
1, 54105
|
| 91 |
+
2, 54105
|
| 92 |
+
3, 54105
|
| 93 |
+
4, 53817
|
| 94 |
+
5, 54105
|
| 95 |
+
6, 54105
|
| 96 |
+
7, 54105
|
| 97 |
+
0, 54457
|
| 98 |
+
1, 54477
|
| 99 |
+
2, 54459
|
| 100 |
+
3, 54459
|
| 101 |
+
4, 54157
|
| 102 |
+
5, 54431
|
| 103 |
+
6, 54451
|
| 104 |
+
7, 54453
|
| 105 |
+
0, 55543
|
| 106 |
+
1, 55543
|
| 107 |
+
2, 55543
|
| 108 |
+
3, 55543
|
| 109 |
+
4, 55255
|
| 110 |
+
5, 55543
|
| 111 |
+
6, 55543
|
| 112 |
+
7, 55543
|
| 113 |
+
0, 56071
|
| 114 |
+
1, 56071
|
| 115 |
+
2, 56071
|
| 116 |
+
3, 56071
|
| 117 |
+
4, 55783
|
| 118 |
+
5, 56071
|
| 119 |
+
6, 56071
|
| 120 |
+
7, 56071
|
| 121 |
+
0, 57013
|
| 122 |
+
1, 57013
|
| 123 |
+
2, 57013
|
| 124 |
+
3, 57013
|
| 125 |
+
4, 56725
|
| 126 |
+
5, 57013
|
| 127 |
+
6, 57013
|
| 128 |
+
7, 57013
|
| 129 |
+
0, 57655
|
| 130 |
+
1, 57655
|
| 131 |
+
2, 57655
|
| 132 |
+
3, 57655
|
| 133 |
+
4, 57367
|
| 134 |
+
5, 57655
|
| 135 |
+
6, 57655
|
| 136 |
+
7, 57655
|
| 137 |
+
0, 96665
|
| 138 |
+
1, 96677
|
| 139 |
+
2, 96663
|
| 140 |
+
3, 96673
|
| 141 |
+
4, 96683
|
| 142 |
+
5, 96663
|
| 143 |
+
6, 96637
|
| 144 |
+
7, 96643
|
| 145 |
+
0, 97161
|
| 146 |
+
1, 97157
|
| 147 |
+
2, 97157
|
| 148 |
+
3, 97157
|
| 149 |
+
4, 97177
|
| 150 |
+
5, 97157
|
| 151 |
+
6, 97157
|
| 152 |
+
7, 97157
|
| 153 |
+
0, 97163
|
| 154 |
+
1, 97159
|
| 155 |
+
2, 97159
|
| 156 |
+
3, 97159
|
| 157 |
+
4, 97179
|
| 158 |
+
5, 97159
|
| 159 |
+
6, 97159
|
| 160 |
+
7, 97159
|
| 161 |
+
0, 97163
|
| 162 |
+
1, 97159
|
| 163 |
+
2, 97159
|
| 164 |
+
3, 97159
|
| 165 |
+
4, 97179
|
| 166 |
+
5, 97159
|
| 167 |
+
6, 97159
|
| 168 |
+
7, 97159
|
| 169 |
+
0, 97163
|
| 170 |
+
1, 97159
|
| 171 |
+
2, 97159
|
| 172 |
+
3, 97159
|
| 173 |
+
4, 97179
|
| 174 |
+
5, 97159
|
| 175 |
+
6, 97159
|
| 176 |
+
7, 97159
|
| 177 |
+
0, 98527
|
| 178 |
+
1, 98523
|
| 179 |
+
2, 98509
|
| 180 |
+
3, 98523
|
| 181 |
+
4, 98543
|
| 182 |
+
5, 98509
|
| 183 |
+
6, 98509
|
| 184 |
+
7, 98523
|
| 185 |
+
0, 102247
|
| 186 |
+
1, 101859
|
| 187 |
+
2, 102243
|
| 188 |
+
3, 101859
|
| 189 |
+
4, 102263
|
| 190 |
+
5, 102243
|
| 191 |
+
6, 101859
|
| 192 |
+
7, 101859
|
| 193 |
+
0, 108151
|
| 194 |
+
1, 107703
|
| 195 |
+
2, 108077
|
| 196 |
+
3, 107737
|
| 197 |
+
4, 108237
|
| 198 |
+
5, 108049
|
| 199 |
+
6, 107681
|
| 200 |
+
7, 107659
|
| 201 |
+
0, 132791
|
| 202 |
+
1, 132535
|
| 203 |
+
2, 132919
|
| 204 |
+
3, 132535
|
| 205 |
+
4, 132939
|
| 206 |
+
5, 132919
|
| 207 |
+
6, 132535
|
| 208 |
+
7, 132535
|
| 209 |
+
0, 132791
|
| 210 |
+
1, 132535
|
| 211 |
+
2, 132919
|
| 212 |
+
3, 132535
|
| 213 |
+
4, 132939
|
| 214 |
+
5, 132919
|
| 215 |
+
6, 132535
|
| 216 |
+
7, 132535
|
| 217 |
+
0, 132789
|
| 218 |
+
1, 5886
|
| 219 |
+
2, 132917
|
| 220 |
+
3, 132533
|
| 221 |
+
4, 132937
|
| 222 |
+
5, 132917
|
| 223 |
+
6, 132533
|
| 224 |
+
7, 132533
|
| 225 |
+
0, 3963
|
| 226 |
+
1, 3965
|
| 227 |
+
2, 3966
|
| 228 |
+
3, 3966
|
| 229 |
+
4, 3966
|
| 230 |
+
5, 4286
|
| 231 |
+
6, 132533
|
| 232 |
+
7, 5566
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=QWEN3_235B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=token-drop-analysis
|
| 6 |
+
commit=b099fd84
|
| 7 |
+
num_layers=12
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=12288
|
| 10 |
+
moe_ffn_hidden=1536
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=18874368
|
| 15 |
+
heads=64
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=12
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=0
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=65536
|
| 39 |
+
mean_tokens_per_expert=4096
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=16
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.03125
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=1
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv
CHANGED
|
@@ -14,7 +14,7 @@
|
|
| 14 |
5, 0
|
| 15 |
6, 0
|
| 16 |
7, 0
|
| 17 |
-
0,
|
| 18 |
1, 1
|
| 19 |
2, 1
|
| 20 |
3, 1
|
|
@@ -42,7 +42,7 @@
|
|
| 42 |
1, 4
|
| 43 |
2, 4
|
| 44 |
3, 4
|
| 45 |
-
4,
|
| 46 |
5, 4
|
| 47 |
6, 4
|
| 48 |
7, 4
|
|
@@ -62,14 +62,14 @@
|
|
| 62 |
5, 1975
|
| 63 |
6, 1975
|
| 64 |
7, 1975
|
| 65 |
-
0,
|
| 66 |
-
1,
|
| 67 |
-
2,
|
| 68 |
-
3,
|
| 69 |
-
4,
|
| 70 |
-
5,
|
| 71 |
-
6,
|
| 72 |
-
7,
|
| 73 |
0, 48319
|
| 74 |
1, 48319
|
| 75 |
2, 48319
|
|
@@ -85,7 +85,7 @@
|
|
| 85 |
4, 49639
|
| 86 |
5, 49927
|
| 87 |
6, 49927
|
| 88 |
-
7,
|
| 89 |
0, 53697
|
| 90 |
1, 53697
|
| 91 |
2, 53697
|
|
@@ -102,14 +102,14 @@
|
|
| 102 |
5, 54105
|
| 103 |
6, 54105
|
| 104 |
7, 54105
|
| 105 |
-
0,
|
| 106 |
-
1,
|
| 107 |
-
2,
|
| 108 |
-
3,
|
| 109 |
-
4,
|
| 110 |
-
5,
|
| 111 |
-
6,
|
| 112 |
-
7,
|
| 113 |
0, 55543
|
| 114 |
1, 55543
|
| 115 |
2, 55543
|
|
@@ -142,14 +142,14 @@
|
|
| 142 |
5, 59799
|
| 143 |
6, 59799
|
| 144 |
7, 59799
|
| 145 |
-
0,
|
| 146 |
-
1,
|
| 147 |
-
2,
|
| 148 |
-
3,
|
| 149 |
-
4,
|
| 150 |
-
5,
|
| 151 |
-
6,
|
| 152 |
-
7,
|
| 153 |
0, 106307
|
| 154 |
1, 106347
|
| 155 |
2, 106347
|
|
@@ -182,62 +182,62 @@
|
|
| 182 |
5, 106349
|
| 183 |
6, 106349
|
| 184 |
7, 106349
|
| 185 |
-
0,
|
| 186 |
-
1,
|
| 187 |
-
2,
|
| 188 |
-
3,
|
| 189 |
4, 110381
|
| 190 |
5, 110401
|
| 191 |
6, 110401
|
| 192 |
-
7,
|
| 193 |
0, 111009
|
| 194 |
-
1,
|
| 195 |
2, 111049
|
| 196 |
3, 111049
|
| 197 |
4, 111029
|
| 198 |
-
5,
|
| 199 |
-
6,
|
| 200 |
-
7,
|
| 201 |
-
0,
|
| 202 |
-
1,
|
| 203 |
-
2,
|
| 204 |
-
3,
|
| 205 |
-
4,
|
| 206 |
-
5,
|
| 207 |
-
6,
|
| 208 |
-
7,
|
| 209 |
0, 141439
|
| 210 |
-
1,
|
| 211 |
2, 141559
|
| 212 |
3, 141559
|
| 213 |
4, 141539
|
| 214 |
-
5,
|
| 215 |
-
6,
|
| 216 |
-
7,
|
| 217 |
0, 141439
|
| 218 |
-
1,
|
| 219 |
2, 141559
|
| 220 |
3, 141559
|
| 221 |
4, 141539
|
| 222 |
-
5,
|
| 223 |
-
6,
|
| 224 |
-
7,
|
| 225 |
0, 141439
|
| 226 |
-
1,
|
| 227 |
2, 141559
|
| 228 |
3, 141559
|
| 229 |
4, 141539
|
| 230 |
-
5,
|
| 231 |
-
6,
|
| 232 |
-
7,
|
| 233 |
0, 140191
|
| 234 |
-
1,
|
| 235 |
2, 140311
|
| 236 |
3, 140311
|
| 237 |
-
4,
|
| 238 |
-
5,
|
| 239 |
-
6,
|
| 240 |
-
7,
|
| 241 |
0, 4643
|
| 242 |
1, 3366
|
| 243 |
2, 3366
|
|
@@ -245,7 +245,7 @@
|
|
| 245 |
4, 3366
|
| 246 |
5, 3366
|
| 247 |
6, 3622
|
| 248 |
-
7,
|
| 249 |
0, 0
|
| 250 |
1, 0
|
| 251 |
2, 0
|
|
|
|
| 14 |
5, 0
|
| 15 |
6, 0
|
| 16 |
7, 0
|
| 17 |
+
0, 1
|
| 18 |
1, 1
|
| 19 |
2, 1
|
| 20 |
3, 1
|
|
|
|
| 42 |
1, 4
|
| 43 |
2, 4
|
| 44 |
3, 4
|
| 45 |
+
4, 45
|
| 46 |
5, 4
|
| 47 |
6, 4
|
| 48 |
7, 4
|
|
|
|
| 62 |
5, 1975
|
| 63 |
6, 1975
|
| 64 |
7, 1975
|
| 65 |
+
0, 45207
|
| 66 |
+
1, 45495
|
| 67 |
+
2, 45111
|
| 68 |
+
3, 45159
|
| 69 |
+
4, 44823
|
| 70 |
+
5, 45063
|
| 71 |
+
6, 44775
|
| 72 |
+
7, 45063
|
| 73 |
0, 48319
|
| 74 |
1, 48319
|
| 75 |
2, 48319
|
|
|
|
| 85 |
4, 49639
|
| 86 |
5, 49927
|
| 87 |
6, 49927
|
| 88 |
+
7, 49929
|
| 89 |
0, 53697
|
| 90 |
1, 53697
|
| 91 |
2, 53697
|
|
|
|
| 102 |
5, 54105
|
| 103 |
6, 54105
|
| 104 |
7, 54105
|
| 105 |
+
0, 54439
|
| 106 |
+
1, 54415
|
| 107 |
+
2, 54433
|
| 108 |
+
3, 54427
|
| 109 |
+
4, 54123
|
| 110 |
+
5, 54447
|
| 111 |
+
6, 54409
|
| 112 |
+
7, 54417
|
| 113 |
0, 55543
|
| 114 |
1, 55543
|
| 115 |
2, 55543
|
|
|
|
| 142 |
5, 59799
|
| 143 |
6, 59799
|
| 144 |
7, 59799
|
| 145 |
+
0, 105853
|
| 146 |
+
1, 105915
|
| 147 |
+
2, 105931
|
| 148 |
+
3, 105909
|
| 149 |
+
4, 105833
|
| 150 |
+
5, 105867
|
| 151 |
+
6, 105931
|
| 152 |
+
7, 105889
|
| 153 |
0, 106307
|
| 154 |
1, 106347
|
| 155 |
2, 106347
|
|
|
|
| 182 |
5, 106349
|
| 183 |
6, 106349
|
| 184 |
7, 106349
|
| 185 |
+
0, 107673
|
| 186 |
+
1, 110401
|
| 187 |
+
2, 108481
|
| 188 |
+
3, 108481
|
| 189 |
4, 110381
|
| 190 |
5, 110401
|
| 191 |
6, 110401
|
| 192 |
+
7, 108481
|
| 193 |
0, 111009
|
| 194 |
+
1, 111049
|
| 195 |
2, 111049
|
| 196 |
3, 111049
|
| 197 |
4, 111029
|
| 198 |
+
5, 111433
|
| 199 |
+
6, 111433
|
| 200 |
+
7, 111049
|
| 201 |
+
0, 111665
|
| 202 |
+
1, 111671
|
| 203 |
+
2, 111699
|
| 204 |
+
3, 111667
|
| 205 |
+
4, 111663
|
| 206 |
+
5, 112029
|
| 207 |
+
6, 112079
|
| 208 |
+
7, 111679
|
| 209 |
0, 141439
|
| 210 |
+
1, 141559
|
| 211 |
2, 141559
|
| 212 |
3, 141559
|
| 213 |
4, 141539
|
| 214 |
+
5, 141943
|
| 215 |
+
6, 141943
|
| 216 |
+
7, 141559
|
| 217 |
0, 141439
|
| 218 |
+
1, 141559
|
| 219 |
2, 141559
|
| 220 |
3, 141559
|
| 221 |
4, 141539
|
| 222 |
+
5, 141943
|
| 223 |
+
6, 141943
|
| 224 |
+
7, 141559
|
| 225 |
0, 141439
|
| 226 |
+
1, 141559
|
| 227 |
2, 141559
|
| 228 |
3, 141559
|
| 229 |
4, 141539
|
| 230 |
+
5, 141943
|
| 231 |
+
6, 141943
|
| 232 |
+
7, 141559
|
| 233 |
0, 140191
|
| 234 |
+
1, 140311
|
| 235 |
2, 140311
|
| 236 |
3, 140311
|
| 237 |
+
4, 4902
|
| 238 |
+
5, 140695
|
| 239 |
+
6, 140695
|
| 240 |
+
7, 140311
|
| 241 |
0, 4643
|
| 242 |
1, 3366
|
| 243 |
2, 3366
|
|
|
|
| 245 |
4, 3366
|
| 246 |
5, 3366
|
| 247 |
6, 3622
|
| 248 |
+
7, 140311
|
| 249 |
0, 0
|
| 250 |
1, 0
|
| 251 |
2, 0
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 41
|
| 43 |
+
2, 4
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1831
|
| 58 |
+
1, 1831
|
| 59 |
+
2, 1831
|
| 60 |
+
3, 1831
|
| 61 |
+
4, 1831
|
| 62 |
+
5, 1831
|
| 63 |
+
6, 1831
|
| 64 |
+
7, 1831
|
| 65 |
+
0, 39169
|
| 66 |
+
1, 38995
|
| 67 |
+
2, 39197
|
| 68 |
+
3, 38995
|
| 69 |
+
4, 38995
|
| 70 |
+
5, 39197
|
| 71 |
+
6, 38995
|
| 72 |
+
7, 39197
|
| 73 |
+
0, 44847
|
| 74 |
+
1, 44869
|
| 75 |
+
2, 44851
|
| 76 |
+
3, 44811
|
| 77 |
+
4, 44843
|
| 78 |
+
5, 44869
|
| 79 |
+
6, 44841
|
| 80 |
+
7, 44893
|
| 81 |
+
0, 45895
|
| 82 |
+
1, 45897
|
| 83 |
+
2, 45907
|
| 84 |
+
3, 45869
|
| 85 |
+
4, 45893
|
| 86 |
+
5, 45907
|
| 87 |
+
6, 45897
|
| 88 |
+
7, 45907
|
| 89 |
+
0, 46917
|
| 90 |
+
1, 46919
|
| 91 |
+
2, 46929
|
| 92 |
+
3, 46891
|
| 93 |
+
4, 46915
|
| 94 |
+
5, 46929
|
| 95 |
+
6, 46919
|
| 96 |
+
7, 46929
|
| 97 |
+
0, 47013
|
| 98 |
+
1, 47015
|
| 99 |
+
2, 47025
|
| 100 |
+
3, 46987
|
| 101 |
+
4, 47011
|
| 102 |
+
5, 47025
|
| 103 |
+
6, 47015
|
| 104 |
+
7, 47025
|
| 105 |
+
0, 47317
|
| 106 |
+
1, 47311
|
| 107 |
+
2, 47329
|
| 108 |
+
3, 47291
|
| 109 |
+
4, 47315
|
| 110 |
+
5, 47311
|
| 111 |
+
6, 47319
|
| 112 |
+
7, 47329
|
| 113 |
+
0, 47877
|
| 114 |
+
1, 47879
|
| 115 |
+
2, 47889
|
| 116 |
+
3, 47851
|
| 117 |
+
4, 47875
|
| 118 |
+
5, 47889
|
| 119 |
+
6, 47879
|
| 120 |
+
7, 47889
|
| 121 |
+
0, 48013
|
| 122 |
+
1, 48015
|
| 123 |
+
2, 48025
|
| 124 |
+
3, 47987
|
| 125 |
+
4, 48011
|
| 126 |
+
5, 48025
|
| 127 |
+
6, 48015
|
| 128 |
+
7, 48025
|
| 129 |
+
0, 48451
|
| 130 |
+
1, 48449
|
| 131 |
+
2, 48463
|
| 132 |
+
3, 48421
|
| 133 |
+
4, 48445
|
| 134 |
+
5, 48463
|
| 135 |
+
6, 48409
|
| 136 |
+
7, 48463
|
| 137 |
+
0, 48719
|
| 138 |
+
1, 48717
|
| 139 |
+
2, 48727
|
| 140 |
+
3, 48689
|
| 141 |
+
4, 48713
|
| 142 |
+
5, 48727
|
| 143 |
+
6, 48677
|
| 144 |
+
7, 48727
|
| 145 |
+
0, 77033
|
| 146 |
+
1, 77035
|
| 147 |
+
2, 77035
|
| 148 |
+
3, 77055
|
| 149 |
+
4, 77059
|
| 150 |
+
5, 77035
|
| 151 |
+
6, 77031
|
| 152 |
+
7, 77035
|
| 153 |
+
0, 92599
|
| 154 |
+
1, 92601
|
| 155 |
+
2, 92601
|
| 156 |
+
3, 92643
|
| 157 |
+
4, 92647
|
| 158 |
+
5, 92643
|
| 159 |
+
6, 92645
|
| 160 |
+
7, 92611
|
| 161 |
+
0, 92703
|
| 162 |
+
1, 92705
|
| 163 |
+
2, 92705
|
| 164 |
+
3, 92705
|
| 165 |
+
4, 92709
|
| 166 |
+
5, 92705
|
| 167 |
+
6, 92701
|
| 168 |
+
7, 92705
|
| 169 |
+
0, 92705
|
| 170 |
+
1, 92707
|
| 171 |
+
2, 92707
|
| 172 |
+
3, 92707
|
| 173 |
+
4, 92711
|
| 174 |
+
5, 92707
|
| 175 |
+
6, 92703
|
| 176 |
+
7, 92707
|
| 177 |
+
0, 92705
|
| 178 |
+
1, 92707
|
| 179 |
+
2, 92707
|
| 180 |
+
3, 92707
|
| 181 |
+
4, 92711
|
| 182 |
+
5, 92707
|
| 183 |
+
6, 92703
|
| 184 |
+
7, 92707
|
| 185 |
+
0, 92705
|
| 186 |
+
1, 92707
|
| 187 |
+
2, 92707
|
| 188 |
+
3, 92707
|
| 189 |
+
4, 92711
|
| 190 |
+
5, 92707
|
| 191 |
+
6, 92703
|
| 192 |
+
7, 92707
|
| 193 |
+
0, 93353
|
| 194 |
+
1, 93353
|
| 195 |
+
2, 93357
|
| 196 |
+
3, 93395
|
| 197 |
+
4, 93371
|
| 198 |
+
5, 93331
|
| 199 |
+
6, 93337
|
| 200 |
+
7, 93335
|
| 201 |
+
0, 94065
|
| 202 |
+
1, 94067
|
| 203 |
+
2, 94259
|
| 204 |
+
3, 94835
|
| 205 |
+
4, 94263
|
| 206 |
+
5, 94259
|
| 207 |
+
6, 94255
|
| 208 |
+
7, 94163
|
| 209 |
+
0, 94705
|
| 210 |
+
1, 94729
|
| 211 |
+
2, 94883
|
| 212 |
+
3, 95463
|
| 213 |
+
4, 94901
|
| 214 |
+
5, 94881
|
| 215 |
+
6, 94899
|
| 216 |
+
7, 94821
|
| 217 |
+
0, 124273
|
| 218 |
+
1, 124261
|
| 219 |
+
2, 124477
|
| 220 |
+
3, 125063
|
| 221 |
+
4, 124455
|
| 222 |
+
5, 124477
|
| 223 |
+
6, 124443
|
| 224 |
+
7, 124381
|
| 225 |
+
0, 124273
|
| 226 |
+
1, 124261
|
| 227 |
+
2, 124477
|
| 228 |
+
3, 125063
|
| 229 |
+
4, 124455
|
| 230 |
+
5, 124477
|
| 231 |
+
6, 124443
|
| 232 |
+
7, 124381
|
| 233 |
+
0, 124273
|
| 234 |
+
1, 124261
|
| 235 |
+
2, 124477
|
| 236 |
+
3, 125063
|
| 237 |
+
4, 124455
|
| 238 |
+
5, 124477
|
| 239 |
+
6, 124443
|
| 240 |
+
7, 124381
|
| 241 |
+
0, 124273
|
| 242 |
+
1, 124261
|
| 243 |
+
2, 124477
|
| 244 |
+
3, 125063
|
| 245 |
+
4, 124455
|
| 246 |
+
5, 124477
|
| 247 |
+
6, 124443
|
| 248 |
+
7, 124381
|
| 249 |
+
0, 124273
|
| 250 |
+
1, 124261
|
| 251 |
+
2, 124477
|
| 252 |
+
3, 125063
|
| 253 |
+
4, 124455
|
| 254 |
+
5, 124477
|
| 255 |
+
6, 124443
|
| 256 |
+
7, 124381
|
| 257 |
+
0, 124273
|
| 258 |
+
1, 124261
|
| 259 |
+
2, 124477
|
| 260 |
+
3, 125063
|
| 261 |
+
4, 124455
|
| 262 |
+
5, 124477
|
| 263 |
+
6, 124443
|
| 264 |
+
7, 124381
|
| 265 |
+
0, 906
|
| 266 |
+
1, 906
|
| 267 |
+
2, 906
|
| 268 |
+
3, 906
|
| 269 |
+
4, 1226
|
| 270 |
+
5, 124307
|
| 271 |
+
6, 2506
|
| 272 |
+
7, 906
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=QWEN3_30B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=token-drop-analysis
|
| 6 |
+
commit=b099fd84
|
| 7 |
+
num_layers=48
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=6144
|
| 10 |
+
moe_ffn_hidden=768
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=4718592
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=48
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=0
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=4096
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=32768
|
| 39 |
+
mean_tokens_per_expert=2048
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=16
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.05859375
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/train.log
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:13812f8ee67386cd1c544bc6e126525ff567e1ac8bb6a759965cd7145a4f0d1a
|
| 3 |
+
size 13017463
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv
CHANGED
|
@@ -14,7 +14,7 @@
|
|
| 14 |
5, 0
|
| 15 |
6, 0
|
| 16 |
7, 0
|
| 17 |
-
0,
|
| 18 |
1, 1
|
| 19 |
2, 1
|
| 20 |
3, 1
|
|
@@ -40,52 +40,52 @@
|
|
| 40 |
7, 4
|
| 41 |
0, 4
|
| 42 |
1, 4
|
| 43 |
-
2,
|
| 44 |
3, 4
|
| 45 |
4, 4
|
| 46 |
5, 4
|
| 47 |
-
6,
|
| 48 |
-
7,
|
| 49 |
-
0,
|
| 50 |
-
1,
|
| 51 |
-
2,
|
| 52 |
-
3,
|
| 53 |
-
4,
|
| 54 |
-
5,
|
| 55 |
-
6,
|
| 56 |
-
7,
|
| 57 |
-
0,
|
| 58 |
-
1,
|
| 59 |
-
2,
|
| 60 |
-
3,
|
| 61 |
-
4,
|
| 62 |
-
5,
|
| 63 |
-
6,
|
| 64 |
-
7,
|
| 65 |
-
0,
|
| 66 |
-
1,
|
| 67 |
-
2,
|
| 68 |
-
3,
|
| 69 |
-
4,
|
| 70 |
-
5,
|
| 71 |
-
6,
|
| 72 |
-
7,
|
| 73 |
-
0,
|
| 74 |
-
1,
|
| 75 |
-
2,
|
| 76 |
-
3,
|
| 77 |
-
4,
|
| 78 |
-
5,
|
| 79 |
-
6,
|
| 80 |
-
7,
|
| 81 |
-
0,
|
| 82 |
-
1,
|
| 83 |
-
2,
|
| 84 |
-
3,
|
| 85 |
-
4,
|
| 86 |
-
5,
|
| 87 |
-
6,
|
| 88 |
-
7,
|
| 89 |
0, 46917
|
| 90 |
1, 46919
|
| 91 |
2, 46929
|
|
@@ -102,14 +102,14 @@
|
|
| 102 |
5, 47025
|
| 103 |
6, 47015
|
| 104 |
7, 47025
|
| 105 |
-
0,
|
| 106 |
-
1,
|
| 107 |
-
2,
|
| 108 |
-
3,
|
| 109 |
-
4,
|
| 110 |
-
5,
|
| 111 |
-
6,
|
| 112 |
-
7,
|
| 113 |
0, 47877
|
| 114 |
1, 47879
|
| 115 |
2, 47889
|
|
@@ -142,14 +142,14 @@
|
|
| 142 |
5, 49145
|
| 143 |
6, 49117
|
| 144 |
7, 49145
|
| 145 |
-
0,
|
| 146 |
-
1,
|
| 147 |
-
2,
|
| 148 |
3, 100647
|
| 149 |
-
4,
|
| 150 |
5, 100639
|
| 151 |
6, 100639
|
| 152 |
-
7,
|
| 153 |
0, 101347
|
| 154 |
1, 101369
|
| 155 |
2, 101353
|
|
@@ -182,91 +182,83 @@
|
|
| 182 |
5, 101355
|
| 183 |
6, 101355
|
| 184 |
7, 101355
|
| 185 |
-
0,
|
| 186 |
-
1,
|
| 187 |
-
2,
|
| 188 |
-
3,
|
| 189 |
-
4,
|
| 190 |
-
5,
|
| 191 |
-
6,
|
| 192 |
-
7,
|
| 193 |
-
0,
|
| 194 |
-
1,
|
| 195 |
-
2,
|
| 196 |
-
3,
|
| 197 |
-
4,
|
| 198 |
-
5,
|
| 199 |
-
6,
|
| 200 |
-
7,
|
| 201 |
-
0,
|
| 202 |
-
1,
|
| 203 |
-
2,
|
| 204 |
-
3,
|
| 205 |
-
4,
|
| 206 |
-
5,
|
| 207 |
-
6,
|
| 208 |
-
7,
|
| 209 |
-
0,
|
| 210 |
-
1,
|
| 211 |
-
2,
|
| 212 |
-
3,
|
| 213 |
-
4,
|
| 214 |
-
5,
|
| 215 |
-
6,
|
| 216 |
-
7,
|
| 217 |
-
0,
|
| 218 |
-
1,
|
| 219 |
-
2,
|
| 220 |
-
3,
|
| 221 |
-
4,
|
| 222 |
-
5,
|
| 223 |
-
6,
|
| 224 |
-
7,
|
| 225 |
-
0,
|
| 226 |
-
1,
|
| 227 |
-
2,
|
| 228 |
-
3,
|
| 229 |
-
4,
|
| 230 |
-
5,
|
| 231 |
-
6,
|
| 232 |
-
7,
|
| 233 |
-
0,
|
| 234 |
-
1,
|
| 235 |
-
2,
|
| 236 |
-
3,
|
| 237 |
-
4,
|
| 238 |
-
5,
|
| 239 |
-
6,
|
| 240 |
-
7,
|
| 241 |
-
0,
|
| 242 |
-
1,
|
| 243 |
-
2,
|
| 244 |
-
3,
|
| 245 |
-
4,
|
| 246 |
-
5,
|
| 247 |
-
6,
|
| 248 |
-
7,
|
| 249 |
-
0,
|
| 250 |
-
1,
|
| 251 |
-
2,
|
| 252 |
-
3,
|
| 253 |
-
4,
|
| 254 |
-
5,
|
| 255 |
-
6,
|
| 256 |
-
7,
|
| 257 |
-
0,
|
| 258 |
-
1,
|
| 259 |
-
2,
|
| 260 |
-
3,
|
| 261 |
-
4,
|
| 262 |
-
5,
|
| 263 |
-
6,
|
| 264 |
-
7,
|
| 265 |
-
0, 132130
|
| 266 |
-
1, 2183
|
| 267 |
-
2, 903
|
| 268 |
-
3, 903
|
| 269 |
-
4, 903
|
| 270 |
-
5, 903
|
| 271 |
-
6, 905
|
| 272 |
-
7, 1162
|
|
|
|
| 14 |
5, 0
|
| 15 |
6, 0
|
| 16 |
7, 0
|
| 17 |
+
0, 4
|
| 18 |
1, 1
|
| 19 |
2, 1
|
| 20 |
3, 1
|
|
|
|
| 40 |
7, 4
|
| 41 |
0, 4
|
| 42 |
1, 4
|
| 43 |
+
2, 4
|
| 44 |
3, 4
|
| 45 |
4, 4
|
| 46 |
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1537
|
| 58 |
+
1, 1537
|
| 59 |
+
2, 1537
|
| 60 |
+
3, 1537
|
| 61 |
+
4, 1537
|
| 62 |
+
5, 1537
|
| 63 |
+
6, 1537
|
| 64 |
+
7, 1537
|
| 65 |
+
0, 1975
|
| 66 |
+
1, 1975
|
| 67 |
+
2, 1975
|
| 68 |
+
3, 1975
|
| 69 |
+
4, 1975
|
| 70 |
+
5, 1975
|
| 71 |
+
6, 1975
|
| 72 |
+
7, 1975
|
| 73 |
+
0, 44783
|
| 74 |
+
1, 44819
|
| 75 |
+
2, 44805
|
| 76 |
+
3, 44799
|
| 77 |
+
4, 44801
|
| 78 |
+
5, 44813
|
| 79 |
+
6, 44775
|
| 80 |
+
7, 44799
|
| 81 |
+
0, 45561
|
| 82 |
+
1, 45563
|
| 83 |
+
2, 45553
|
| 84 |
+
3, 45535
|
| 85 |
+
4, 45549
|
| 86 |
+
5, 45583
|
| 87 |
+
6, 45573
|
| 88 |
+
7, 45593
|
| 89 |
0, 46917
|
| 90 |
1, 46919
|
| 91 |
2, 46929
|
|
|
|
| 102 |
5, 47025
|
| 103 |
6, 47015
|
| 104 |
7, 47025
|
| 105 |
+
0, 47217
|
| 106 |
+
1, 47173
|
| 107 |
+
2, 47195
|
| 108 |
+
3, 47167
|
| 109 |
+
4, 47201
|
| 110 |
+
5, 47201
|
| 111 |
+
6, 47185
|
| 112 |
+
7, 47183
|
| 113 |
0, 47877
|
| 114 |
1, 47879
|
| 115 |
2, 47889
|
|
|
|
| 142 |
5, 49145
|
| 143 |
6, 49117
|
| 144 |
7, 49145
|
| 145 |
+
0, 100697
|
| 146 |
+
1, 100719
|
| 147 |
+
2, 100639
|
| 148 |
3, 100647
|
| 149 |
+
4, 100651
|
| 150 |
5, 100639
|
| 151 |
6, 100639
|
| 152 |
+
7, 100639
|
| 153 |
0, 101347
|
| 154 |
1, 101369
|
| 155 |
2, 101353
|
|
|
|
| 182 |
5, 101355
|
| 183 |
6, 101355
|
| 184 |
7, 101355
|
| 185 |
+
0, 101891
|
| 186 |
+
1, 101973
|
| 187 |
+
2, 101989
|
| 188 |
+
3, 101947
|
| 189 |
+
4, 101965
|
| 190 |
+
5, 101925
|
| 191 |
+
6, 101951
|
| 192 |
+
7, 101955
|
| 193 |
+
0, 102709
|
| 194 |
+
1, 102923
|
| 195 |
+
2, 103483
|
| 196 |
+
3, 103491
|
| 197 |
+
4, 102919
|
| 198 |
+
5, 102811
|
| 199 |
+
6, 102715
|
| 200 |
+
7, 102715
|
| 201 |
+
0, 103339
|
| 202 |
+
1, 103489
|
| 203 |
+
2, 104065
|
| 204 |
+
3, 104051
|
| 205 |
+
4, 103483
|
| 206 |
+
5, 103405
|
| 207 |
+
6, 103283
|
| 208 |
+
7, 103271
|
| 209 |
+
0, 132757
|
| 210 |
+
1, 132957
|
| 211 |
+
2, 133539
|
| 212 |
+
3, 133527
|
| 213 |
+
4, 132993
|
| 214 |
+
5, 132867
|
| 215 |
+
6, 132751
|
| 216 |
+
7, 132771
|
| 217 |
+
0, 132757
|
| 218 |
+
1, 132957
|
| 219 |
+
2, 133539
|
| 220 |
+
3, 133527
|
| 221 |
+
4, 132993
|
| 222 |
+
5, 132867
|
| 223 |
+
6, 132751
|
| 224 |
+
7, 132771
|
| 225 |
+
0, 132757
|
| 226 |
+
1, 132957
|
| 227 |
+
2, 133539
|
| 228 |
+
3, 133527
|
| 229 |
+
4, 132993
|
| 230 |
+
5, 132867
|
| 231 |
+
6, 132751
|
| 232 |
+
7, 132771
|
| 233 |
+
0, 132757
|
| 234 |
+
1, 132957
|
| 235 |
+
2, 133539
|
| 236 |
+
3, 133527
|
| 237 |
+
4, 132993
|
| 238 |
+
5, 132867
|
| 239 |
+
6, 132751
|
| 240 |
+
7, 132771
|
| 241 |
+
0, 132757
|
| 242 |
+
1, 132957
|
| 243 |
+
2, 133539
|
| 244 |
+
3, 133527
|
| 245 |
+
4, 132993
|
| 246 |
+
5, 132867
|
| 247 |
+
6, 132751
|
| 248 |
+
7, 132771
|
| 249 |
+
0, 132587
|
| 250 |
+
1, 132787
|
| 251 |
+
2, 133369
|
| 252 |
+
3, 133357
|
| 253 |
+
4, 132823
|
| 254 |
+
5, 132697
|
| 255 |
+
6, 132581
|
| 256 |
+
7, 132601
|
| 257 |
+
0, 1162
|
| 258 |
+
1, 132141
|
| 259 |
+
2, 2186
|
| 260 |
+
3, 906
|
| 261 |
+
4, 906
|
| 262 |
+
5, 906
|
| 263 |
+
6, 906
|
| 264 |
+
7, 906
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b3cc59379cd8890580a7c6d403bc7e87dbf0a1ff3a78a68cd8dc0e79d3aae086
|
| 3 |
+
size 13020062
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv
CHANGED
|
@@ -25,7 +25,7 @@
|
|
| 25 |
0, 4
|
| 26 |
1, 4
|
| 27 |
2, 4
|
| 28 |
-
3,
|
| 29 |
4, 1
|
| 30 |
5, 1
|
| 31 |
6, 1
|
|
@@ -38,38 +38,38 @@
|
|
| 38 |
5, 4
|
| 39 |
6, 4
|
| 40 |
7, 4
|
| 41 |
-
0,
|
| 42 |
-
1,
|
| 43 |
-
2,
|
| 44 |
-
3,
|
| 45 |
-
4,
|
| 46 |
-
5,
|
| 47 |
-
6,
|
| 48 |
-
7,
|
| 49 |
-
0,
|
| 50 |
-
1,
|
| 51 |
-
2,
|
| 52 |
-
3,
|
| 53 |
-
4,
|
| 54 |
-
5,
|
| 55 |
-
6,
|
| 56 |
-
7,
|
| 57 |
-
0,
|
| 58 |
-
1,
|
| 59 |
-
2,
|
| 60 |
-
3,
|
| 61 |
-
4,
|
| 62 |
-
5,
|
| 63 |
-
6,
|
| 64 |
-
7,
|
| 65 |
-
0,
|
| 66 |
-
1,
|
| 67 |
-
2,
|
| 68 |
-
3,
|
| 69 |
-
4,
|
| 70 |
-
5,
|
| 71 |
-
6,
|
| 72 |
-
7,
|
| 73 |
0, 45247
|
| 74 |
1, 45249
|
| 75 |
2, 45259
|
|
@@ -79,10 +79,10 @@
|
|
| 79 |
6, 45249
|
| 80 |
7, 45259
|
| 81 |
0, 45895
|
| 82 |
-
1,
|
| 83 |
2, 45907
|
| 84 |
3, 45869
|
| 85 |
-
4,
|
| 86 |
5, 45907
|
| 87 |
6, 45897
|
| 88 |
7, 45907
|
|
@@ -102,14 +102,14 @@
|
|
| 102 |
5, 47025
|
| 103 |
6, 47015
|
| 104 |
7, 47025
|
| 105 |
-
0,
|
| 106 |
-
1,
|
| 107 |
-
2,
|
| 108 |
-
3,
|
| 109 |
-
4,
|
| 110 |
-
5,
|
| 111 |
-
6,
|
| 112 |
-
7,
|
| 113 |
0, 47877
|
| 114 |
1, 47879
|
| 115 |
2, 47889
|
|
@@ -182,86 +182,86 @@
|
|
| 182 |
5, 101355
|
| 183 |
6, 101355
|
| 184 |
7, 101355
|
| 185 |
-
0,
|
| 186 |
-
1,
|
| 187 |
-
2,
|
| 188 |
-
3,
|
| 189 |
-
4,
|
| 190 |
-
5,
|
| 191 |
-
6,
|
| 192 |
-
7,
|
| 193 |
0, 103477
|
| 194 |
-
1,
|
| 195 |
-
2,
|
| 196 |
3, 103491
|
| 197 |
-
4,
|
| 198 |
-
5,
|
| 199 |
-
6,
|
| 200 |
-
7,
|
| 201 |
-
0,
|
| 202 |
-
1,
|
| 203 |
-
2,
|
| 204 |
3, 104141
|
| 205 |
-
4,
|
| 206 |
-
5,
|
| 207 |
-
6,
|
| 208 |
-
7,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
0, 133525
|
| 210 |
-
1,
|
| 211 |
-
2,
|
| 212 |
3, 133527
|
| 213 |
-
4,
|
| 214 |
-
5,
|
| 215 |
-
6,
|
| 216 |
-
7,
|
| 217 |
0, 133525
|
| 218 |
-
1,
|
| 219 |
-
2,
|
| 220 |
3, 133527
|
| 221 |
-
4,
|
| 222 |
-
5,
|
| 223 |
-
6,
|
| 224 |
-
7,
|
| 225 |
0, 133525
|
| 226 |
-
1,
|
| 227 |
-
2,
|
| 228 |
3, 133527
|
| 229 |
-
4,
|
| 230 |
-
5,
|
| 231 |
-
6,
|
| 232 |
-
7,
|
| 233 |
0, 133525
|
| 234 |
-
1,
|
| 235 |
-
2,
|
| 236 |
3, 133527
|
| 237 |
-
4,
|
| 238 |
-
5,
|
| 239 |
-
6,
|
| 240 |
-
7,
|
| 241 |
0, 133525
|
| 242 |
-
1,
|
| 243 |
-
2,
|
| 244 |
3, 133527
|
| 245 |
-
4,
|
| 246 |
-
5,
|
| 247 |
-
6,
|
| 248 |
-
7,
|
| 249 |
-
0, 133291
|
| 250 |
-
1, 133101
|
| 251 |
-
2, 132601
|
| 252 |
-
3, 133095
|
| 253 |
-
4, 132631
|
| 254 |
-
5, 132601
|
| 255 |
-
6, 132773
|
| 256 |
-
7, 132601
|
| 257 |
0, 132709
|
| 258 |
-
1,
|
| 259 |
-
2,
|
| 260 |
-
3,
|
| 261 |
-
4,
|
| 262 |
-
5,
|
| 263 |
-
6,
|
| 264 |
-
7,
|
| 265 |
0, 0
|
| 266 |
1, 0
|
| 267 |
2, 0
|
|
|
|
| 25 |
0, 4
|
| 26 |
1, 4
|
| 27 |
2, 4
|
| 28 |
+
3, 4
|
| 29 |
4, 1
|
| 30 |
5, 1
|
| 31 |
6, 1
|
|
|
|
| 38 |
5, 4
|
| 39 |
6, 4
|
| 40 |
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 4
|
| 43 |
+
2, 4
|
| 44 |
+
3, 629
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1831
|
| 58 |
+
1, 1831
|
| 59 |
+
2, 1831
|
| 60 |
+
3, 1831
|
| 61 |
+
4, 1831
|
| 62 |
+
5, 1831
|
| 63 |
+
6, 1831
|
| 64 |
+
7, 1831
|
| 65 |
+
0, 39169
|
| 66 |
+
1, 38995
|
| 67 |
+
2, 39197
|
| 68 |
+
3, 38995
|
| 69 |
+
4, 38995
|
| 70 |
+
5, 39197
|
| 71 |
+
6, 38995
|
| 72 |
+
7, 39197
|
| 73 |
0, 45247
|
| 74 |
1, 45249
|
| 75 |
2, 45259
|
|
|
|
| 79 |
6, 45249
|
| 80 |
7, 45259
|
| 81 |
0, 45895
|
| 82 |
+
1, 45897
|
| 83 |
2, 45907
|
| 84 |
3, 45869
|
| 85 |
+
4, 45893
|
| 86 |
5, 45907
|
| 87 |
6, 45897
|
| 88 |
7, 45907
|
|
|
|
| 102 |
5, 47025
|
| 103 |
6, 47015
|
| 104 |
7, 47025
|
| 105 |
+
0, 47243
|
| 106 |
+
1, 47263
|
| 107 |
+
2, 47227
|
| 108 |
+
3, 47201
|
| 109 |
+
4, 47221
|
| 110 |
+
5, 47243
|
| 111 |
+
6, 47241
|
| 112 |
+
7, 47235
|
| 113 |
0, 47877
|
| 114 |
1, 47879
|
| 115 |
2, 47889
|
|
|
|
| 182 |
5, 101355
|
| 183 |
6, 101355
|
| 184 |
7, 101355
|
| 185 |
+
0, 101965
|
| 186 |
+
1, 101987
|
| 187 |
+
2, 101995
|
| 188 |
+
3, 102033
|
| 189 |
+
4, 102013
|
| 190 |
+
5, 101961
|
| 191 |
+
6, 101937
|
| 192 |
+
7, 101977
|
| 193 |
0, 103477
|
| 194 |
+
1, 102731
|
| 195 |
+
2, 103483
|
| 196 |
3, 103491
|
| 197 |
+
4, 103015
|
| 198 |
+
5, 103483
|
| 199 |
+
6, 103291
|
| 200 |
+
7, 103291
|
| 201 |
+
0, 104139
|
| 202 |
+
1, 103385
|
| 203 |
+
2, 104175
|
| 204 |
3, 104141
|
| 205 |
+
4, 103665
|
| 206 |
+
5, 104111
|
| 207 |
+
6, 103931
|
| 208 |
+
7, 103909
|
| 209 |
+
0, 133525
|
| 210 |
+
1, 132765
|
| 211 |
+
2, 133539
|
| 212 |
+
3, 133527
|
| 213 |
+
4, 133089
|
| 214 |
+
5, 133539
|
| 215 |
+
6, 133327
|
| 216 |
+
7, 133347
|
| 217 |
0, 133525
|
| 218 |
+
1, 132765
|
| 219 |
+
2, 133539
|
| 220 |
3, 133527
|
| 221 |
+
4, 133089
|
| 222 |
+
5, 133539
|
| 223 |
+
6, 133327
|
| 224 |
+
7, 133347
|
| 225 |
0, 133525
|
| 226 |
+
1, 132765
|
| 227 |
+
2, 133539
|
| 228 |
3, 133527
|
| 229 |
+
4, 133089
|
| 230 |
+
5, 133539
|
| 231 |
+
6, 133327
|
| 232 |
+
7, 133347
|
| 233 |
0, 133525
|
| 234 |
+
1, 132765
|
| 235 |
+
2, 133539
|
| 236 |
3, 133527
|
| 237 |
+
4, 133089
|
| 238 |
+
5, 133539
|
| 239 |
+
6, 133327
|
| 240 |
+
7, 133347
|
| 241 |
0, 133525
|
| 242 |
+
1, 132765
|
| 243 |
+
2, 133539
|
| 244 |
3, 133527
|
| 245 |
+
4, 133089
|
| 246 |
+
5, 133539
|
| 247 |
+
6, 133327
|
| 248 |
+
7, 133347
|
| 249 |
0, 133525
|
| 250 |
+
1, 132765
|
| 251 |
+
2, 133539
|
| 252 |
3, 133527
|
| 253 |
+
4, 133089
|
| 254 |
+
5, 133539
|
| 255 |
+
6, 133327
|
| 256 |
+
7, 133347
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 257 |
0, 132709
|
| 258 |
+
1, 131949
|
| 259 |
+
2, 2442
|
| 260 |
+
3, 132711
|
| 261 |
+
4, 2442
|
| 262 |
+
5, 132723
|
| 263 |
+
6, 132511
|
| 264 |
+
7, 132531
|
| 265 |
0, 0
|
| 266 |
1, 0
|
| 267 |
2, 0
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:79645046d96acc4d96e5897dd6ab188f7aa4c5a066c78e9ad5c0517f53376cdb
|
| 3 |
+
size 15163552
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv
CHANGED
|
@@ -1,3 +1,8 @@
|
|
| 1 |
model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
|
| 2 |
-
QWEN3_235B echo measure 2.00 OK 141943 1 0.0 1.
|
| 3 |
-
QWEN3_235B echo validate 1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
|
| 2 |
+
QWEN3_235B echo measure 2.00 OK 141943 1 0.0 1.03125 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
|
| 3 |
+
QWEN3_235B echo validate 1.03125 OK 132927 1 0.0 1.029296875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1
|
| 4 |
+
QWEN3_235B echo forced 1.03125 OK 132939 1 0.0 1.025390625 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1
|
| 5 |
+
QWEN3_30B echo measure 2.00 OK 133539 1 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
|
| 6 |
+
QWEN3_30B echo validate 1.05859375 OK 125063 0 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1
|
| 7 |
+
QWEN3_30B echo forced 2.00 OK 133539 1 0.0 1.0546875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1
|
| 8 |
+
DSV2_LITE_L24 echo measure 2.00 OK 140961 1 0.0 1.0234375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv
CHANGED
|
@@ -1 +1,3 @@
|
|
| 1 |
model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
|
|
|
|
|
|
|
|
|
| 1 |
model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
|
| 2 |
+
QWEN3_235B echo SELECTED 1.03125 1.029296875 0.0 132927 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1
|
| 3 |
+
QWEN3_30B echo SELECTED 2.00 1.05859375 0.0 133539 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
|
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv
CHANGED
|
@@ -1,3 +1,8 @@
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 2 |
-
QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK
|
| 3 |
-
QWEN3_235B echo matched 16 8192 8 1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 2 |
+
QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 444.56 440.90 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
|
| 3 |
+
QWEN3_235B echo matched 16 8192 8 1.03125 none 1 OK 434.67 425.80 132927 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1
|
| 4 |
+
QWEN3_235B echo matched 16 8192 8 1.03125 none+forcebal 1 OK 419.26 407.60 132939 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1
|
| 5 |
+
QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 965.97 915.30 133539 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
|
| 6 |
+
QWEN3_30B echo matched 16 4096 8 1.05859375 none 1 OK 1018.41 969.70 125063 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1
|
| 7 |
+
QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 1021.91 944.00 133539 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1
|
| 8 |
+
DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 841.21 834.60 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
|
tierb_study_20260826_144644/hf_watcher.log
CHANGED
|
@@ -55,3 +55,6 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 55 |
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 56 |
Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
|
| 57 |
[2026-08-26T17:09:12+00:00] backup pass
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 56 |
Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
|
| 57 |
[2026-08-26T17:09:12+00:00] backup pass
|
| 58 |
+
uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
|
| 59 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 60 |
+
[2026-08-26T17:19:17+00:00] backup pass
|
tierb_study_20260826_144644/study.log
CHANGED
|
@@ -1378,3 +1378,47 @@ driver_repo: /workspace/blackwell_run_scripts @ 6553011
|
|
| 1378 |
=== STAGE 7 -- Tier B probe and zero-drop capacity selection
|
| 1379 |
|
| 1380 |
=== STEP echo__tierbprobe__gbs8
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1378 |
=== STAGE 7 -- Tier B probe and zero-drop capacity selection
|
| 1379 |
|
| 1380 |
=== STEP echo__tierbprobe__gbs8
|
| 1381 |
+
==============================================================
|
| 1382 |
+
CAMPAIGN: echo backend=echo phase=tierbprobe
|
| 1383 |
+
Megatron: /workspace/Megatron-MoE-Echo @ token-drop-analysis
|
| 1384 |
+
GBS=8 seq=4096 repeats=3 nsys=1
|
| 1385 |
+
results -> /workspace/tierb_study_20260826_144644/echo/tierbprobe
|
| 1386 |
+
==============================================================
|
| 1387 |
+
--------------------------------------------------------------------------
|
| 1388 |
+
[17:09:26] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16)
|
| 1389 |
+
[guard] iter 11/12 (70s)
|
| 1390 |
+
[guard] iter 12/12 (80s)
|
| 1391 |
+
[17:10:47] QWEN3_235B echo: measured required=1.03125 at C=2.00 -> validating C=1.03125
|
| 1392 |
+
--------------------------------------------------------------------------
|
| 1393 |
+
[17:10:47] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125 (slots=16)
|
| 1394 |
+
[guard] iter 10/12 (70s)
|
| 1395 |
+
[guard] iter 12/12 (80s)
|
| 1396 |
+
--------------------------------------------------------------------------
|
| 1397 |
+
[17:12:07] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal (slots=16)
|
| 1398 |
+
[guard] iter 12/12 (70s)
|
| 1399 |
+
[guard] tearing down process group (outcome=DONE)
|
| 1400 |
+
--------------------------------------------------------------------------
|
| 1401 |
+
[17:13:24] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00 (slots=16)
|
| 1402 |
+
[guard] iter 5/12 (70s)
|
| 1403 |
+
[guard] iter 12/12 (80s)
|
| 1404 |
+
[guard] tearing down process group (outcome=DONE)
|
| 1405 |
+
[17:14:49] QWEN3_30B echo: measured required=1.05859375 at C=2.00 -> validating C=1.05859375
|
| 1406 |
+
--------------------------------------------------------------------------
|
| 1407 |
+
[17:14:49] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375 (slots=16)
|
| 1408 |
+
[guard] iter 3/12 (70s)
|
| 1409 |
+
[guard] iter 12/12 (80s)
|
| 1410 |
+
[guard] tearing down process group (outcome=DONE)
|
| 1411 |
+
[17:16:16] QWEN3_30B echo: C=1.05859375 dropped; keeping C=2.00
|
| 1412 |
+
--------------------------------------------------------------------------
|
| 1413 |
+
[17:16:16] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal (slots=16)
|
| 1414 |
+
[guard] iter 5/12 (70s)
|
| 1415 |
+
[guard] iter 12/12 (80s)
|
| 1416 |
+
[guard] tearing down process group (outcome=DONE)
|
| 1417 |
+
--------------------------------------------------------------------------
|
| 1418 |
+
[17:17:41] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 (slots=8)
|
| 1419 |
+
[guard] iter 6/12 (70s)
|
| 1420 |
+
[guard] iter 12/12 (80s)
|
| 1421 |
+
[guard] tearing down process group (outcome=DONE)
|
| 1422 |
+
[17:19:04] DSV2_LITE_L24 echo: measured required=1.0234375 at C=2.00 -> validating C=1.0234375
|
| 1423 |
+
--------------------------------------------------------------------------
|
| 1424 |
+
[17:19:04] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375 (slots=8)
|