AbhishekGhosh1998 commited on
Commit
c913fb8
·
verified ·
1 Parent(s): 9c7614b

granularity results tierb_study_20260826_144644 @ 2026-08-26T15:52:41Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
Files changed (31) hide show
  1. tierb_study_20260826_144644/.completed/echo__tierbprobe__gbs8 +1 -1
  2. tierb_study_20260826_144644/STATUS.txt +3 -2
  3. tierb_study_20260826_144644/hf_watcher.log +3 -0
  4. tierb_study_20260826_144644/steps.tsv +1 -0
  5. tierb_study_20260826_144644/study.log +132 -0
  6. tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt +2 -2
  7. tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt +3 -3
  8. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv +99 -99
  9. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log +0 -0
  10. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +232 -0
  11. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env +61 -0
  12. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log +0 -0
  13. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv +124 -84
  14. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env +1 -1
  15. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log +0 -0
  16. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +64 -0
  17. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env +61 -0
  18. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log +647 -0
  19. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv +120 -136
  20. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log +0 -0
  21. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +208 -0
  22. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env +61 -0
  23. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log +0 -0
  24. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv +172 -164
  25. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log +0 -0
  26. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +240 -0
  27. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env +61 -0
  28. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log +0 -0
  29. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +6 -6
  30. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +3 -4
  31. tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv +7 -7
tierb_study_20260826_144644/.completed/echo__tierbprobe__gbs8 CHANGED
@@ -1 +1 @@
1
- 2026-08-26T14:57:54+00:00
 
1
+ 2026-08-26T15:43:24+00:00
tierb_study_20260826_144644/STATUS.txt CHANGED
@@ -1,9 +1,10 @@
1
  study_root: /workspace/tierb_study_20260826_144644
2
  gbs: 8
3
- updated: 2026-08-26T15:32:33+00:00
4
- current: echo__tierbprobe__gbs8
5
 
6
  completed steps:
 
7
 
8
  failed steps:
9
  FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe
 
1
  study_root: /workspace/tierb_study_20260826_144644
2
  gbs: 8
3
+ updated: 2026-08-26T15:43:26+00:00
4
+ current: ultraep__tierbprobe__gbs8
5
 
6
  completed steps:
7
+ echo__tierbprobe__gbs8
8
 
9
  failed steps:
10
  FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe
tierb_study_20260826_144644/hf_watcher.log CHANGED
@@ -20,3 +20,6 @@ Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
20
  uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
21
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
22
  [2026-08-26T15:42:37+00:00] backup pass
 
 
 
 
20
  uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
21
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
22
  [2026-08-26T15:42:37+00:00] backup pass
23
+ uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
24
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
25
+ [2026-08-26T15:52:40+00:00] backup pass
tierb_study_20260826_144644/steps.tsv CHANGED
@@ -2,3 +2,4 @@ status campaign phase gbs started ended result_dir
2
  OK echo tierbprobe 8 2026-08-26T14:46:44+00:00 2026-08-26T14:57:54+00:00 /workspace/tierb_study_20260826_144644/echo/tierbprobe
3
  FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe
4
  FAIL moonep tierbprobe 8 2026-08-26T15:05:24+00:00 2026-08-26T15:12:52+00:00 /workspace/tierb_study_20260826_144644/moonep/tierbprobe
 
 
2
  OK echo tierbprobe 8 2026-08-26T14:46:44+00:00 2026-08-26T14:57:54+00:00 /workspace/tierb_study_20260826_144644/echo/tierbprobe
3
  FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe
4
  FAIL moonep tierbprobe 8 2026-08-26T15:05:24+00:00 2026-08-26T15:12:52+00:00 /workspace/tierb_study_20260826_144644/moonep/tierbprobe
5
+ OK echo tierbprobe 8 2026-08-26T15:32:33+00:00 2026-08-26T15:43:24+00:00 /workspace/tierb_study_20260826_144644/echo/tierbprobe
tierb_study_20260826_144644/study.log CHANGED
@@ -503,3 +503,135 @@ driver_repo: /workspace/blackwell_run_scripts @ d11391f
503
  [guard] tearing down process group (outcome=DONE)
504
  --------------------------------------------------------------------------
505
  [15:42:07] CELL QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal (slots=8)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
503
  [guard] tearing down process group (outcome=DONE)
504
  --------------------------------------------------------------------------
505
  [15:42:07] CELL QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal (slots=8)
506
+ [guard] iter 12/12 (70s)
507
+ [guard] tearing down process group (outcome=DONE)
508
+ --------------------------------------------------------------------------
509
+
510
+ ==============================================================
511
+ SWEEP COMPLETE ok=8 failed=0
512
+ ==============================================================
513
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
514
+ QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 441.20 435.60 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
515
+ QWEN3_235B echo matched 16 8192 8 2.00 none+forcebal 1 OK 422.89 416.80 141939 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1
516
+ QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 1052.93 996.60 133569 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
517
+ QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 963.67 920.30 133533 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1
518
+ DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 806.56 800.50 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
519
+ DSV2_LITE_L24 echo matched 8 8192 8 2.00 none+forcebal 1 OK 799.97 793.80 140541 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1
520
+ QWEN2_57B echo matched 8 4096 8 2.00 none 1 OK 286.46 282.70 109133 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1
521
+ QWEN2_57B echo matched 8 4096 8 2.00 none+forcebal 1 OK 276.56 271.10 109131 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1
522
+
523
+ Results: /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv
524
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8
525
+ ==============================================================
526
+ 8 probe runs -> /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv
527
+ run arm cap rawExpI preRankI postRankI drop% reqCap
528
+ --------------------------------------------------------------------------------------------------------------------
529
+ DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 natural 2.00 7.4425* 2.1173 1.0000 1.0000
530
+ DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 forced 2.00 1.0287 1.0061 1.0000 1.0000
531
+ QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep natural 2.00 6.8085* 2.1623 1.0000 1.0000
532
+ QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__fo forced 2.00 1.0352 1.0074 1.0000 1.0000
533
+ QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/re natural 2.00 15.4584* 2.5319 1.0000 1.0000
534
+ QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__f forced 2.00 1.0390 1.0053 1.0000 1.0000
535
+ QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep natural 2.00 14.5598* 2.3229 1.0000 1.0000
536
+ QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__fo forced 2.00 1.0565 1.0077 1.0000 1.0000
537
+
538
+ * At least one globally aggregated snapshot reached the E/K expert-Imax ceiling.
539
+ Inspect raw_expert_imax_mean and hottest_expert_token_fraction_mean before generalising.
540
+
541
+ ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional.
542
+ The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally.
543
+
544
+ TIER B PROBE -- what these logs contain, per backend
545
+ ----------------------------------------------------
546
+ Capacity matrix:
547
+ ECHO: start C=2.00; on OOM descend 1.75, 1.50, 1.25. Stop at
548
+ the highest fitting zero-drop capacity. If a fitting capacity drops,
549
+ smaller values cannot help and the model is blocked.
550
+ UltraEP: measure required_capacity at C=1.25, round it UP to the next
551
+ 0.01, then validate that capacity once.
552
+ MoonEP: natural + forced, no capacity factor.
553
+
554
+ The forced arm runs once at the selected/high-headroom capacity. It validates
555
+ instrumentation and is not part of capacity selection.
556
+
557
+ Each backend instruments a different thing:
558
+
559
+ echo local raw-router histograms and local post-reroute placement.
560
+ reduce_probe.py aligns and sums all eight source ranks before
561
+ calculating raw_expert_*, pre_home_rank_*,
562
+ post_physical_rank_* and valid_max_rank_reduction_*.
563
+
564
+ ultraep global pre/post rank vectors plus explicit drop accounting:
565
+ drop_rate_pct and required_capacity -- the capacity factor the run
566
+ WOULD have needed. The vectors are duplicated identically on all
567
+ logging ranks and are deduplicated, not summed.
568
+
569
+ moonep local raw-router histograms, actual payload rows per physical
570
+ destination, and the bytes-copy side. The reducer globally sums
571
+ each (iteration, layer) snapshot first. ECHO reports no comparable
572
+ byte counters, so MoonEP bytes are a self-comparison across models,
573
+ never a cross-backend ranking.
574
+
575
+ Do NOT quote ECHO's logged pre_echo imbalance_ratio or
576
+ max_load_reduction_pct. The branch mixes source-rank and destination-rank
577
+ axes; the reducer marks both invalid and reconstructs the comparison globally.
578
+
579
+ Two boundary cases:
580
+ raw_expert_imax_max == E/K
581
+ -> at least one global snapshot had one expert in every
582
+ token's top-K set. Inspect the MEAN before generalising.
583
+ required_capacity or min-safe-capacity at the configured capacity
584
+ -> clipped by the buffer; it means "at or over", not
585
+ "fits comfortably".
586
+
587
+ tools/reduce_probe.py has already written probe_summary.tsv and .json here.
588
+
589
+ Campaign results: /workspace/tierb_study_20260826_144644/echo/tierbprobe
590
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tierb_study_20260826_144644/echo/tierbprobe --compare
591
+ Already logged in to Hugging Face as: AbhishekGhosh1998
592
+ uploaded: /workspace/tierb_study_20260826_144644/echo/tierbprobe -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644/echo/tierbprobe
593
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
594
+
595
+ === STEP ultraep__tierbprobe__gbs8
596
+ ==============================================================
597
+ CAMPAIGN: ultraep backend=ultraep phase=tierbprobe
598
+ Megatron: /workspace/Megatron-UltraEP @ ultraep-token-drop-analysis
599
+ GBS=8 seq=4096 repeats=3 nsys=1
600
+ results -> /workspace/tierb_study_20260826_144644/ultraep/tierbprobe
601
+ ==============================================================
602
+ --------------------------------------------------------------------------
603
+ [15:43:42] CELL QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25 (slots=16)
604
+ [guard] iter 9/12 (60s)
605
+ [guard] iter 12/12 (70s)
606
+ [guard] tearing down process group (outcome=DONE)
607
+ --------------------------------------------------------------------------
608
+ [15:44:54] CELL QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal (slots=16)
609
+ [guard] iter 12/12 (60s)
610
+ [guard] tearing down process group (outcome=DONE)
611
+ --------------------------------------------------------------------------
612
+ [15:46:01] CELL QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25 (slots=16)
613
+ [guard] iter 4/12 (60s)
614
+ [guard] iter 12/12 (70s)
615
+ [guard] tearing down process group (outcome=DONE)
616
+ --------------------------------------------------------------------------
617
+ [15:47:16] CELL QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal (slots=16)
618
+ [guard] iter 3/12 (60s)
619
+ [guard] iter 12/12 (70s)
620
+ [guard] tearing down process group (outcome=DONE)
621
+ --------------------------------------------------------------------------
622
+ [15:48:32] CELL DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25 (slots=8)
623
+ [guard] iter 4/12 (60s)
624
+ [guard] iter 12/12 (70s)
625
+ [guard] tearing down process group (outcome=DONE)
626
+ --------------------------------------------------------------------------
627
+ [15:49:47] CELL DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal (slots=8)
628
+ [guard] iter 5/12 (60s)
629
+ [guard] iter 12/12 (70s)
630
+ [guard] tearing down process group (outcome=DONE)
631
+ --------------------------------------------------------------------------
632
+ [15:51:01] CELL QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25 (slots=8)
633
+ [guard] iter 12/12 (60s)
634
+ [guard] tearing down process group (outcome=DONE)
635
+ --------------------------------------------------------------------------
636
+ [15:52:03] CELL QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal (slots=8)
637
+ [15:52:06] waiting for stale ranks to exit ...
tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt CHANGED
@@ -1,9 +1,9 @@
1
- timestamp: 2026-08-26T14:58:12+00:00
2
  host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
3
  gpu: NVIDIA B200 x8
4
  driver: 580.95.05
5
  study_scripts_branch: granularity-ultraep
6
- study_scripts_commit: 81efb04
7
  campaign: ultraep
8
  backend: ultraep
9
  phase: tierbprobe
 
1
+ timestamp: 2026-08-26T15:43:41+00:00
2
  host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
3
  gpu: NVIDIA B200 x8
4
  driver: 580.95.05
5
  study_scripts_branch: granularity-ultraep
6
+ study_scripts_commit: 05e7365
7
  campaign: ultraep
8
  backend: ultraep
9
  phase: tierbprobe
tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt CHANGED
@@ -1,7 +1,7 @@
1
  ==============================================================
2
  PROVENANCE: ultraep__tierbprobe__gbs8
3
  ==============================================================
4
- captured_at : 2026-08-26T14:57:57+00:00
5
  hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
6
  kernel : 6.8.0-1043-nvidia
7
  cwd : /workspace/blackwell_run_scripts
@@ -21,7 +21,7 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_
21
  1, 120 MHz, 1965 MHz, 1000.00 W, 34
22
  2, 120 MHz, 1965 MHz, 1000.00 W, 34
23
  3, 120 MHz, 1965 MHz, 1000.00 W, 30
24
- 4, 120 MHz, 1965 MHz, 1000.00 W, 29
25
  5, 120 MHz, 1965 MHz, 1000.00 W, 34
26
  6, 120 MHz, 1965 MHz, 1000.00 W, 34
27
  7, 120 MHz, 1965 MHz, 1000.00 W, 29
@@ -64,7 +64,7 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_
64
  UltraEP main 94cab099b44f
65
  DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
  MoonEP megatron-integration c27d35da15f2
67
- blackwell_run_scripts granularity-ultraep 81efb04918d8
68
 
69
  ---- environment ---------------------------------------------
70
  CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
 
1
  ==============================================================
2
  PROVENANCE: ultraep__tierbprobe__gbs8
3
  ==============================================================
4
+ captured_at : 2026-08-26T15:43:26+00:00
5
  hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
6
  kernel : 6.8.0-1043-nvidia
7
  cwd : /workspace/blackwell_run_scripts
 
21
  1, 120 MHz, 1965 MHz, 1000.00 W, 34
22
  2, 120 MHz, 1965 MHz, 1000.00 W, 34
23
  3, 120 MHz, 1965 MHz, 1000.00 W, 30
24
+ 4, 120 MHz, 1965 MHz, 1000.00 W, 30
25
  5, 120 MHz, 1965 MHz, 1000.00 W, 34
26
  6, 120 MHz, 1965 MHz, 1000.00 W, 34
27
  7, 120 MHz, 1965 MHz, 1000.00 W, 29
 
64
  UltraEP main 94cab099b44f
65
  DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
  MoonEP megatron-integration c27d35da15f2
67
+ blackwell_run_scripts granularity-ultraep 05e7365f983f
68
 
69
  ---- environment ---------------------------------------------
70
  CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 1
18
  1, 1
19
  2, 1
20
  3, 1
@@ -26,7 +26,7 @@
26
  1, 4
27
  2, 4
28
  3, 4
29
- 4, 1
30
  5, 1
31
  6, 1
32
  7, 1
@@ -38,14 +38,14 @@
38
  5, 4
39
  6, 4
40
  7, 4
41
- 0, 761
42
- 1, 725
43
- 2, 737
44
- 3, 725
45
- 4, 725
46
- 5, 725
47
- 6, 747
48
- 7, 725
49
  0, 1537
50
  1, 1537
51
  2, 1537
@@ -62,14 +62,14 @@
62
  5, 2471
63
  6, 2471
64
  7, 2471
65
- 0, 2871
66
- 1, 2871
67
- 2, 2871
68
- 3, 2871
69
- 4, 2871
70
- 5, 2871
71
- 6, 2871
72
- 7, 2871
73
  0, 28477
74
  1, 28565
75
  2, 28411
@@ -81,19 +81,19 @@
81
  0, 34335
82
  1, 34419
83
  2, 34269
84
- 3, 34267
85
- 4, 34267
86
- 5, 34265
87
  6, 34401
88
  7, 34419
89
- 0, 39471
90
- 1, 39555
91
- 2, 39405
92
- 3, 39405
93
- 4, 39405
94
- 5, 39401
95
- 6, 39537
96
- 7, 39555
97
  0, 39771
98
  1, 39855
99
  2, 39705
@@ -110,14 +110,6 @@
110
  5, 39701
111
  6, 39837
112
  7, 39855
113
- 0, 39779
114
- 1, 39863
115
- 2, 39713
116
- 3, 39713
117
- 4, 39713
118
- 5, 39709
119
- 6, 39845
120
- 7, 39863
121
  0, 39781
122
  1, 39865
123
  2, 39715
@@ -126,14 +118,22 @@
126
  5, 39711
127
  6, 39847
128
  7, 39865
129
- 0, 111665
130
- 1, 111781
131
- 2, 111693
132
- 3, 111607
133
- 4, 111671
134
- 5, 111589
135
- 6, 111749
136
- 7, 111745
 
 
 
 
 
 
 
 
137
  0, 112149
138
  1, 112235
139
  2, 112149
@@ -150,38 +150,30 @@
150
  5, 112085
151
  6, 112221
152
  7, 112239
153
- 0, 112155
154
- 1, 112239
155
- 2, 112153
156
- 3, 112089
157
- 4, 112153
158
- 5, 112085
159
- 6, 112221
160
- 7, 112239
161
- 0, 114517
162
- 1, 112583
163
- 2, 114539
164
- 3, 114495
165
- 4, 114525
166
- 5, 114443
167
- 6, 112557
168
- 7, 112611
169
- 0, 115425
170
- 1, 113483
171
- 2, 115419
172
- 3, 115389
173
- 4, 115449
174
- 5, 115383
175
- 6, 113435
176
- 7, 113495
177
- 0, 127845
178
- 1, 129971
179
- 2, 127845
180
- 3, 129825
181
- 4, 129889
182
- 5, 129817
183
- 6, 130023
184
- 7, 129971
185
  0, 127845
186
  1, 129971
187
  2, 127845
@@ -206,27 +198,35 @@
206
  5, 129817
207
  6, 130023
208
  7, 129971
209
- 0, 126899
210
- 1, 129025
211
- 2, 126899
212
- 3, 128879
213
- 4, 128943
214
- 5, 128871
215
- 6, 129077
216
- 7, 129025
217
- 0, 124071
218
- 1, 125963
219
  2, 124071
220
  3, 126051
221
- 4, 125727
222
- 5, 125723
223
- 6, 126089
224
- 7, 126197
225
- 0, 321
226
- 1, 4
227
- 2, 4
228
- 3, 4
229
- 4, 4
230
- 5, 4
231
- 6, 68
232
- 7, 124723
 
 
 
 
 
 
 
 
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 4
18
  1, 1
19
  2, 1
20
  3, 1
 
26
  1, 4
27
  2, 4
28
  3, 4
29
+ 4, 4
30
  5, 1
31
  6, 1
32
  7, 1
 
38
  5, 4
39
  6, 4
40
  7, 4
41
+ 0, 821
42
+ 1, 821
43
+ 2, 819
44
+ 3, 819
45
+ 4, 819
46
+ 5, 819
47
+ 6, 819
48
+ 7, 819
49
  0, 1537
50
  1, 1537
51
  2, 1537
 
62
  5, 2471
63
  6, 2471
64
  7, 2471
65
+ 0, 2847
66
+ 1, 2843
67
+ 2, 2851
68
+ 3, 2849
69
+ 4, 2847
70
+ 5, 2867
71
+ 6, 2837
72
+ 7, 2839
73
  0, 28477
74
  1, 28565
75
  2, 28411
 
81
  0, 34335
82
  1, 34419
83
  2, 34269
84
+ 3, 32155
85
+ 4, 34203
86
+ 5, 34201
87
  6, 34401
88
  7, 34419
89
+ 0, 39031
90
+ 1, 39129
91
+ 2, 38987
92
+ 3, 38983
93
+ 4, 38957
94
+ 5, 38979
95
+ 6, 39095
96
+ 7, 39103
97
  0, 39771
98
  1, 39855
99
  2, 39705
 
110
  5, 39701
111
  6, 39837
112
  7, 39855
 
 
 
 
 
 
 
 
113
  0, 39781
114
  1, 39865
115
  2, 39715
 
118
  5, 39711
119
  6, 39847
120
  7, 39865
121
+ 0, 39781
122
+ 1, 39865
123
+ 2, 39715
124
+ 3, 39715
125
+ 4, 39715
126
+ 5, 39711
127
+ 6, 39847
128
+ 7, 39865
129
+ 0, 111693
130
+ 1, 111809
131
+ 2, 111725
132
+ 3, 111649
133
+ 4, 111721
134
+ 5, 111673
135
+ 6, 111821
136
+ 7, 111827
137
  0, 112149
138
  1, 112235
139
  2, 112149
 
150
  5, 112085
151
  6, 112221
152
  7, 112239
153
+ 0, 114405
154
+ 1, 112485
155
+ 2, 114387
156
+ 3, 114353
157
+ 4, 114365
158
+ 5, 114335
159
+ 6, 112375
160
+ 7, 112441
161
+ 0, 115241
162
+ 1, 113277
163
+ 2, 115239
164
+ 3, 115175
165
+ 4, 115239
166
+ 5, 115171
167
+ 6, 113259
168
+ 7, 113277
169
+ 0, 115949
170
+ 1, 114023
171
+ 2, 115985
172
+ 3, 115877
173
+ 4, 115889
174
+ 5, 115853
175
+ 6, 113897
176
+ 7, 113995
 
 
 
 
 
 
 
 
177
  0, 127845
178
  1, 129971
179
  2, 127845
 
198
  5, 129817
199
  6, 130023
200
  7, 129971
201
+ 0, 127545
202
+ 1, 129671
203
+ 2, 127545
204
+ 3, 129525
205
+ 4, 129589
206
+ 5, 129517
207
+ 6, 129723
208
+ 7, 129671
209
+ 0, 124391
210
+ 1, 126197
211
  2, 124071
212
  3, 126051
213
+ 4, 126179
214
+ 5, 126301
215
+ 6, 126633
216
+ 7, 126455
217
+ 0, 122003
218
+ 1, 124129
219
+ 2, 122067
220
+ 3, 124693
221
+ 4, 125339
222
+ 5, 124679
223
+ 6, 124827
224
+ 7, 124449
225
+ 0, 1
226
+ 1, 1
227
+ 2, 3
228
+ 3, 132
229
+ 4, 125337
230
+ 5, 644
231
+ 6, 4
232
+ 7, 4
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 27
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 2225
66
+ 1, 2225
67
+ 2, 2225
68
+ 3, 2225
69
+ 4, 2225
70
+ 5, 2225
71
+ 6, 2225
72
+ 7, 2225
73
+ 0, 9077
74
+ 1, 9157
75
+ 2, 9117
76
+ 3, 8993
77
+ 4, 9077
78
+ 5, 8691
79
+ 6, 8945
80
+ 7, 9005
81
+ 0, 28837
82
+ 1, 28937
83
+ 2, 28815
84
+ 3, 28755
85
+ 4, 28767
86
+ 5, 28749
87
+ 6, 28917
88
+ 7, 28939
89
+ 0, 38971
90
+ 1, 39063
91
+ 2, 38909
92
+ 3, 38901
93
+ 4, 38903
94
+ 5, 38889
95
+ 6, 39009
96
+ 7, 39035
97
+ 0, 39771
98
+ 1, 39855
99
+ 2, 39705
100
+ 3, 39705
101
+ 4, 39705
102
+ 5, 39701
103
+ 6, 39837
104
+ 7, 39855
105
+ 0, 39771
106
+ 1, 39855
107
+ 2, 39705
108
+ 3, 39705
109
+ 4, 39705
110
+ 5, 39701
111
+ 6, 39837
112
+ 7, 39855
113
+ 0, 39781
114
+ 1, 39865
115
+ 2, 39715
116
+ 3, 39715
117
+ 4, 39715
118
+ 5, 39711
119
+ 6, 39847
120
+ 7, 39865
121
+ 0, 39781
122
+ 1, 39865
123
+ 2, 39715
124
+ 3, 39715
125
+ 4, 39715
126
+ 5, 39711
127
+ 6, 39847
128
+ 7, 39865
129
+ 0, 111653
130
+ 1, 111775
131
+ 2, 111677
132
+ 3, 111635
133
+ 4, 111681
134
+ 5, 111613
135
+ 6, 111739
136
+ 7, 111773
137
+ 0, 112149
138
+ 1, 112235
139
+ 2, 112149
140
+ 3, 112083
141
+ 4, 112147
142
+ 5, 112081
143
+ 6, 112215
144
+ 7, 112235
145
+ 0, 112155
146
+ 1, 112239
147
+ 2, 112153
148
+ 3, 112089
149
+ 4, 112153
150
+ 5, 112085
151
+ 6, 112221
152
+ 7, 112239
153
+ 0, 114413
154
+ 1, 112461
155
+ 2, 114397
156
+ 3, 114387
157
+ 4, 114489
158
+ 5, 114523
159
+ 6, 114519
160
+ 7, 112465
161
+ 0, 115241
162
+ 1, 113277
163
+ 2, 115239
164
+ 3, 115175
165
+ 4, 115239
166
+ 5, 115171
167
+ 6, 115307
168
+ 7, 113277
169
+ 0, 115997
170
+ 1, 113979
171
+ 2, 115887
172
+ 3, 115821
173
+ 4, 115955
174
+ 5, 115851
175
+ 6, 115995
176
+ 7, 113961
177
+ 0, 127845
178
+ 1, 127923
179
+ 2, 129893
180
+ 3, 129825
181
+ 4, 129889
182
+ 5, 129817
183
+ 6, 129959
184
+ 7, 129971
185
+ 0, 127845
186
+ 1, 127923
187
+ 2, 129893
188
+ 3, 129825
189
+ 4, 129889
190
+ 5, 129817
191
+ 6, 129959
192
+ 7, 129971
193
+ 0, 127845
194
+ 1, 127923
195
+ 2, 129893
196
+ 3, 129825
197
+ 4, 129889
198
+ 5, 129817
199
+ 6, 129959
200
+ 7, 129971
201
+ 0, 127331
202
+ 1, 127249
203
+ 2, 129593
204
+ 3, 129431
205
+ 4, 129589
206
+ 5, 129397
207
+ 6, 129117
208
+ 7, 129671
209
+ 0, 124071
210
+ 1, 124213
211
+ 2, 126377
212
+ 3, 126309
213
+ 4, 126373
214
+ 5, 126427
215
+ 6, 126613
216
+ 7, 126517
217
+ 0, 122649
218
+ 1, 122465
219
+ 2, 124697
220
+ 3, 124367
221
+ 4, 124693
222
+ 5, 124295
223
+ 6, 124181
224
+ 7, 124775
225
+ 0, 1
226
+ 1, 3
227
+ 2, 4
228
+ 3, 4
229
+ 4, 4
230
+ 5, 4
231
+ 6, 4
232
+ 7, 124127
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-token-drop-analysis
6
+ commit=eaf829e7
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=768
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv CHANGED
@@ -26,7 +26,7 @@
26
  1, 4
27
  2, 4
28
  3, 4
29
- 4, 2
30
  5, 1
31
  6, 1
32
  7, 1
@@ -39,10 +39,10 @@
39
  6, 4
40
  7, 4
41
  0, 4
42
- 1, 4
43
- 2, 4
44
  3, 4
45
- 4, 629
46
  5, 4
47
  6, 4
48
  7, 4
@@ -62,14 +62,14 @@
62
  5, 1803
63
  6, 1803
64
  7, 1803
65
- 0, 2393
66
- 1, 2387
67
- 2, 2371
68
- 3, 2385
69
- 4, 2385
70
- 5, 2397
71
- 6, 2393
72
- 7, 2377
73
  0, 46703
74
  1, 46137
75
  2, 46137
@@ -78,75 +78,115 @@
78
  5, 46837
79
  6, 46837
80
  7, 46837
81
- 0, 47107
82
- 1, 46537
83
- 2, 46537
84
- 3, 47237
85
- 4, 47237
86
- 5, 47237
87
- 6, 47237
88
- 7, 47237
89
- 0, 47497
90
- 1, 46927
91
- 2, 46927
92
- 3, 47627
93
- 4, 47627
94
- 5, 47627
95
- 6, 47627
96
- 7, 47627
97
- 0, 47497
98
- 1, 46927
99
- 2, 46927
100
- 3, 47627
101
- 4, 47627
102
- 5, 47627
103
- 6, 47627
104
- 7, 47627
105
- 0, 47497
106
- 1, 46927
107
- 2, 46927
108
- 3, 47627
109
- 4, 47627
110
- 5, 47627
111
- 6, 47627
112
- 7, 47627
113
- 0, 47497
114
- 1, 46927
115
- 2, 46927
116
- 3, 47627
117
- 4, 47627
118
- 5, 47627
119
- 6, 47627
120
- 7, 47627
121
- 0, 47497
122
- 1, 46927
123
- 2, 46927
124
- 3, 47627
125
- 4, 47627
126
- 5, 47627
127
- 6, 47627
128
- 7, 47627
129
- 0, 46851
130
- 1, 46345
131
- 2, 46927
132
- 3, 47301
133
- 4, 47045
134
- 5, 47627
135
- 6, 47301
136
- 7, 46981
137
- 0, 2561
138
- 1, 2561
139
- 2, 2561
140
- 3, 2625
141
- 4, 46981
142
- 5, 2884
143
- 6, 2564
144
- 7, 2564
145
- 0, 0
146
- 1, 0
147
- 2, 0
148
- 3, 0
149
- 4, 0
150
- 5, 0
151
- 6, 0
152
- 7, 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  1, 4
27
  2, 4
28
  3, 4
29
+ 4, 4
30
  5, 1
31
  6, 1
32
  7, 1
 
39
  6, 4
40
  7, 4
41
  0, 4
42
+ 1, 47
43
+ 2, 629
44
  3, 4
45
+ 4, 4
46
  5, 4
47
  6, 4
48
  7, 4
 
62
  5, 1803
63
  6, 1803
64
  7, 1803
65
+ 0, 2443
66
+ 1, 2443
67
+ 2, 2443
68
+ 3, 2443
69
+ 4, 2441
70
+ 5, 2443
71
+ 6, 2443
72
+ 7, 2443
73
  0, 46703
74
  1, 46137
75
  2, 46137
 
78
  5, 46837
79
  6, 46837
80
  7, 46837
81
+ 0, 47109
82
+ 1, 46539
83
+ 2, 46539
84
+ 3, 47243
85
+ 4, 47239
86
+ 5, 47241
87
+ 6, 47239
88
+ 7, 47239
89
+ 0, 48395
90
+ 1, 47825
91
+ 2, 47825
92
+ 3, 48525
93
+ 4, 48525
94
+ 5, 48525
95
+ 6, 48525
96
+ 7, 48525
97
+ 0, 48675
98
+ 1, 48105
99
+ 2, 48105
100
+ 3, 48805
101
+ 4, 48805
102
+ 5, 48805
103
+ 6, 48805
104
+ 7, 48805
105
+ 0, 48675
106
+ 1, 48105
107
+ 2, 48105
108
+ 3, 48805
109
+ 4, 48805
110
+ 5, 48805
111
+ 6, 48805
112
+ 7, 48805
113
+ 0, 50879
114
+ 1, 50309
115
+ 2, 50309
116
+ 3, 51009
117
+ 4, 51009
118
+ 5, 51009
119
+ 6, 51009
120
+ 7, 51009
121
+ 0, 58343
122
+ 1, 58357
123
+ 2, 58373
124
+ 3, 58517
125
+ 4, 58403
126
+ 5, 58311
127
+ 6, 58441
128
+ 7, 58349
129
+ 0, 74975
130
+ 1, 75009
131
+ 2, 74989
132
+ 3, 75019
133
+ 4, 75019
134
+ 5, 75019
135
+ 6, 75019
136
+ 7, 75019
137
+ 0, 74975
138
+ 1, 75009
139
+ 2, 74989
140
+ 3, 75019
141
+ 4, 75019
142
+ 5, 75019
143
+ 6, 75019
144
+ 7, 75019
145
+ 0, 74975
146
+ 1, 75011
147
+ 2, 74991
148
+ 3, 75021
149
+ 4, 75021
150
+ 5, 75021
151
+ 6, 75021
152
+ 7, 75021
153
+ 0, 74975
154
+ 1, 75011
155
+ 2, 74991
156
+ 3, 75021
157
+ 4, 75021
158
+ 5, 75021
159
+ 6, 75021
160
+ 7, 75021
161
+ 0, 75523
162
+ 1, 75597
163
+ 2, 75527
164
+ 3, 75617
165
+ 4, 75571
166
+ 5, 75555
167
+ 6, 75575
168
+ 7, 75577
169
+ 0, 82201
170
+ 1, 82167
171
+ 2, 82031
172
+ 3, 82077
173
+ 4, 82119
174
+ 5, 82099
175
+ 6, 82083
176
+ 7, 82083
177
+ 0, 97625
178
+ 1, 97675
179
+ 2, 97675
180
+ 3, 97551
181
+ 4, 97551
182
+ 5, 97551
183
+ 6, 97551
184
+ 7, 97551
185
+ 0, 4414
186
+ 1, 2814
187
+ 2, 2814
188
+ 3, 2814
189
+ 4, 2814
190
+ 5, 2814
191
+ 6, 3134
192
+ 7, 97551
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env CHANGED
@@ -28,7 +28,7 @@ router_balancing=aux_loss
28
  aux_loss_coeff=1e-2
29
  router_pre_softmax=1
30
  router_fusion=0
31
- router_dtype=
32
  device_initiated_grouped_gemm=1
33
  norm_epsilon=1e-6
34
  seq_length=4096
 
28
  aux_loss_coeff=1e-2
29
  router_pre_softmax=1
30
  router_fusion=0
31
+ router_dtype=fp32
32
  device_initiated_grouped_gemm=1
33
  norm_epsilon=1e-6
34
  seq_length=4096
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1803
58
+ 1, 1803
59
+ 2, 1803
60
+ 3, 1803
61
+ 4, 1803
62
+ 5, 1803
63
+ 6, 1803
64
+ 7, 1803
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-token-drop-analysis
6
+ commit=eaf829e7
7
+ num_layers=10
8
+ hidden=3584
9
+ ffn_hidden=2560
10
+ moe_ffn_hidden=2560
11
+ num_experts=64
12
+ topk=8
13
+ experts_per_rank=8
14
+ expert_params=27525120
15
+ heads=28
16
+ kv_groups=4
17
+ kv_channels=
18
+ qk_layernorm=0
19
+ shared_ffn=20480
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=10
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=1
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-2
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=512
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log ADDED
@@ -0,0 +1,647 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [15:52:16] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe)
2
+ ==============================================================
3
+ QWEN2_57B / ultraep (slots/rank=8 policy=matched)
4
+ E=64 K=8 H'=2560 m=8
5
+ L=10 seq=4096 gbs=8 (1 microbatches)
6
+ cap_mode=static cap=1.25 cg=0 recompute=none
7
+ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-UltraEP @ eaf829e7
9
+ ==============================================================
10
+ W0826 15:52:17.546000 149908 torch/distributed/run.py:766]
11
+ W0826 15:52:17.546000 149908 torch/distributed/run.py:766] *****************************************
12
+ W0826 15:52:17.546000 149908 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0826 15:52:17.546000 149908 torch/distributed/run.py:766] *****************************************
14
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
15
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
16
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
17
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
18
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
19
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
20
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
21
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
22
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
23
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
24
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
25
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
26
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
27
+ Number of virtual stages per pipeline stage: None
28
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
29
+ using torch.bfloat16 for parameters ...
30
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
31
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
32
+ ------------------------ arguments ------------------------
33
+ account_for_embedding_in_pipeline_split ......... False
34
+ account_for_loss_in_pipeline_split .............. False
35
+ accumulate_allreduce_grads_in_fp32 .............. True
36
+ adam_beta1 ...................................... 0.9
37
+ adam_beta2 ...................................... 0.999
38
+ adam_eps ........................................ 1e-08
39
+ add_bias_linear ................................. False
40
+ add_position_embedding .......................... False
41
+ add_qkv_bias .................................... True
42
+ adlr_autoresume ................................. False
43
+ adlr_autoresume_interval ........................ 1000
44
+ align_grad_reduce ............................... True
45
+ align_param_gather .............................. False
46
+ app_tag_run_name ................................ None
47
+ app_tag_run_version ............................. 0.0.0
48
+ apply_layernorm_1p .............................. False
49
+ apply_query_key_layer_scaling ................... False
50
+ apply_residual_connection_post_layernorm ........ False
51
+ apply_rope_fusion ............................... True
52
+ async_save ...................................... None
53
+ async_tensor_model_parallel_allreduce ........... True
54
+ attention_backend ............................... AttnBackend.auto
55
+ attention_dropout ............................... 0.0
56
+ attention_softmax_in_fp32 ....................... False
57
+ auto_detect_ckpt_format ......................... False
58
+ barrier_with_L1_time ............................ True
59
+ bert_binary_head ................................ True
60
+ bert_embedder_type .............................. megatron
61
+ bert_load ....................................... None
62
+ bf16 ............................................ True
63
+ bias_dropout_fusion ............................. True
64
+ bias_gelu_fusion ................................ False
65
+ bias_swiglu_fusion .............................. True
66
+ biencoder_projection_dim ........................ 0
67
+ biencoder_shared_query_context_model ............ False
68
+ block_data_path ................................. None
69
+ calc_ft_timeouts ................................ False
70
+ calculate_per_token_loss ........................ False
71
+ check_for_large_grads ........................... False
72
+ check_for_nan_in_loss_and_grad .................. True
73
+ check_for_spiky_loss ............................ False
74
+ check_weight_hash_across_dp_replicas_interval ... None
75
+ ckpt_assume_constant_structure .................. False
76
+ ckpt_convert_format ............................. None
77
+ ckpt_convert_save ............................... None
78
+ ckpt_convert_update_legacy_dist_opt_format ...... False
79
+ ckpt_format ..................................... torch_dist
80
+ ckpt_fully_parallel_load ........................ False
81
+ ckpt_fully_parallel_save ........................ True
82
+ ckpt_fully_parallel_save_deprecated ............. False
83
+ ckpt_step ....................................... None
84
+ classes_fraction ................................ 1.0
85
+ clip_grad ....................................... 1.0
86
+ clone_scatter_output_in_embedding ............... True
87
+ config_logger_dir ...............................
88
+ consumed_train_samples .......................... 0
89
+ consumed_valid_samples .......................... 0
90
+ context_parallel_size ........................... 1
91
+ cp_comm_type .................................... ['p2p']
92
+ create_attention_mask_in_dataloader ............. True
93
+ cross_entropy_fusion_impl ....................... native
94
+ cross_entropy_loss_fusion ....................... False
95
+ cuda_graph_scope ................................ full
96
+ cuda_graph_warmup_steps ......................... 3
97
+ data_args_path .................................. None
98
+ data_cache_path ................................. None
99
+ data_parallel_random_init ....................... False
100
+ data_parallel_sharding_strategy ................. no_shard
101
+ data_parallel_size .............................. 8
102
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
103
+ data_per_class_fraction ......................... 1.0
104
+ data_sharding ................................... True
105
+ dataloader_type ................................. single
106
+ ddp_average_in_collective ....................... False
107
+ ddp_bucket_size ................................. None
108
+ ddp_num_buckets ................................. None
109
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
110
+ decoder_first_pipeline_num_layers ............... None
111
+ decoder_last_pipeline_num_layers ................ None
112
+ decoder_num_layers .............................. None
113
+ decoder_seq_length .............................. None
114
+ decoupled_lr .................................... None
115
+ decoupled_min_lr ................................ None
116
+ decrease_batch_size_if_needed ................... False
117
+ defer_embedding_wgrad_compute ................... False
118
+ delay_wgrad_compute ............................. False
119
+ deprecated_use_mcore_models ..................... True
120
+ deterministic_mode .............................. False
121
+ dino_bottleneck_size ............................ 256
122
+ dino_freeze_last_layer .......................... 1
123
+ dino_head_hidden_size ........................... 2048
124
+ dino_local_crops_number ......................... 10
125
+ dino_local_img_size ............................. 96
126
+ dino_norm_last_layer ............................ False
127
+ dino_teacher_temp ............................... 0.07
128
+ dino_warmup_teacher_temp ........................ 0.04
129
+ dino_warmup_teacher_temp_epochs ................. 30
130
+ disable_bf16_reduced_precision_matmul ........... False
131
+ disable_mamba_mem_eff_path ...................... False
132
+ disable_straggler_on_startup .................... False
133
+ dist_ckpt_format_deprecated ..................... None
134
+ dist_ckpt_strictness ............................ assume_ok_unexpected
135
+ distribute_saved_activations .................... False
136
+ distributed_backend ............................. nccl
137
+ distributed_timeout_minutes ..................... 10
138
+ embedding_init_method_std ....................... None
139
+ embedding_path .................................. None
140
+ empty_unused_memory_level ....................... 0
141
+ enable_cuda_graph ............................... False
142
+ enable_experimental ............................. False
143
+ enable_ft_package ............................... False
144
+ enable_gloo_process_groups ...................... True
145
+ enable_msc ...................................... True
146
+ enable_one_logger ............................... True
147
+ encoder_num_layers .............................. 10
148
+ encoder_seq_length .............................. 4096
149
+ end_weight_decay ................................ 0.1
150
+ eod_mask_loss ................................... False
151
+ error_injection_rate ............................ 0
152
+ error_injection_type ............................ transient_error
153
+ eval_interval ................................... 1000000
154
+ eval_iters ...................................... 0
155
+ evidence_data_path .............................. None
156
+ exit_duration_in_mins ........................... None
157
+ exit_interval ................................... None
158
+ exit_on_missing_checkpoint ...................... False
159
+ exit_signal_handler ............................. False
160
+ exp_avg_dtype ................................... torch.float32
161
+ exp_avg_sq_dtype ................................ torch.float32
162
+ expert_model_parallel_size ...................... 8
163
+ expert_tensor_parallel_size ..................... 1
164
+ export_force_local_attention .................... False
165
+ export_kd_cfg ................................... None
166
+ export_kd_teacher_ckpt_format ................... None
167
+ export_kd_teacher_load .......................... None
168
+ export_kv_cache_quant ........................... False
169
+ export_legacy_megatron .......................... False
170
+ export_model_type ............................... GPTModel
171
+ export_moe_apply_probs_on_input ................. False
172
+ export_qk_l2_norm ............................... False
173
+ export_quant_cfg ................................ None
174
+ export_real_quant_cfg ........................... None
175
+ export_te_mcore_model ........................... False
176
+ external_cuda_graph ............................. False
177
+ ffn_hidden_size ................................. 2560
178
+ finetune ........................................ False
179
+ finetune_data_split ............................. train
180
+ finetune_hf_dataset ............................. None
181
+ first_last_layers_bf16 .......................... False
182
+ flash_decode .................................... False
183
+ fp16 ............................................ False
184
+ fp16_lm_cross_entropy ........................... False
185
+ fp32_residual_connection ........................ False
186
+ fp8 ............................................. e4m3
187
+ fp8_amax_compute_algo ........................... most_recent
188
+ fp8_amax_history_len ............................ 1
189
+ fp8_interval .................................... 1
190
+ fp8_margin ...................................... 0
191
+ fp8_param_gather ................................ False
192
+ fp8_recipe ...................................... mxfp8
193
+ fp8_wgrad ....................................... True
194
+ fsdp_double_buffer .............................. False
195
+ global_batch_size ............................... 8
196
+ grad_reduce_in_bf16 ............................. False
197
+ gradient_accumulation_fusion .................... True
198
+ gradient_reduce_div_fusion ...................... True
199
+ group_query_attention ........................... True
200
+ head_lr_mult .................................... 1.0
201
+ heterogeneous_layers_config_encoded_json ........ None
202
+ heterogeneous_layers_config_path ................ None
203
+ hidden_dropout .................................. 0.0
204
+ hidden_size ..................................... 3584
205
+ hierarchical_context_parallel_sizes ............. None
206
+ high_priority_stream_groups ..................... []
207
+ hybrid_attention_ratio .......................... 0.0
208
+ hybrid_mlp_ratio ................................ 0.0
209
+ hybrid_override_pattern ......................... None
210
+ hysteresis ...................................... 2
211
+ ict_head_size ................................... None
212
+ ict_load ........................................ None
213
+ img_h ........................................... 224
214
+ img_w ........................................... 224
215
+ indexer_batch_size .............................. 128
216
+ indexer_log_interval ............................ 1000
217
+ inference_batch_times_seqlen_threshold .......... -1
218
+ inference_dynamic_batching ...................... False
219
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
220
+ inference_dynamic_batching_buffer_overflow_factor None
221
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
222
+ inference_dynamic_batching_chunk_size ........... 256
223
+ inference_dynamic_batching_max_requests_override None
224
+ inference_dynamic_batching_max_tokens_override .. None
225
+ inference_dynamic_batching_num_cuda_graphs ...... 16
226
+ inference_max_batch_size ........................ 8
227
+ inference_max_seq_length ........................ 2560
228
+ inference_rng_tracker ........................... False
229
+ init_method_std ................................. 0.01
230
+ init_method_xavier_uniform ...................... False
231
+ init_model_with_meta_device ..................... False
232
+ initial_loss_scale .............................. 4294967296
233
+ inprocess_active_world_size ..................... 8
234
+ inprocess_barrier_timeout ....................... 120
235
+ inprocess_completion_timeout .................... 120
236
+ inprocess_empty_cuda_cache ...................... False
237
+ inprocess_granularity ........................... node
238
+ inprocess_hard_timeout .......................... 90
239
+ inprocess_heartbeat_interval .................... 30
240
+ inprocess_heartbeat_timeout ..................... 60
241
+ inprocess_last_call_wait ........................ 1
242
+ inprocess_max_iterations ........................ None
243
+ inprocess_monitor_process_interval .............. 1.0
244
+ inprocess_monitor_thread_interval ............... 1.0
245
+ inprocess_progress_watchdog_interval ............ 1.0
246
+ inprocess_restart ............................... False
247
+ inprocess_soft_timeout .......................... 60
248
+ inprocess_termination_grace_time ................ 1
249
+ is_hybrid_model ................................. False
250
+ iter_per_epoch .................................. 1250
251
+ iterations_to_skip .............................. []
252
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
253
+ kitchen_config_file ............................. None
254
+ kitchen_recipe_number ........................... None
255
+ kv_channels ..................................... 128
256
+ kv_lora_rank .................................... 32
257
+ lazy_mpu_init ................................... None
258
+ load ............................................ None
259
+ load_main_params_from_ckpt ...................... None
260
+ load_model_opt_format ........................... False
261
+ local_rank ...................................... 0
262
+ log_energy ...................................... False
263
+ log_interval .................................... 1
264
+ log_loss_scale_to_tensorboard ................... True
265
+ log_memory_to_tensorboard ....................... False
266
+ log_num_zeros_in_grad ........................... False
267
+ log_params_norm ................................. False
268
+ log_progress .................................... False
269
+ log_straggler ................................... False
270
+ log_throughput .................................. True
271
+ log_timers_to_tensorboard ....................... False
272
+ log_validation_ppl_to_tensorboard ............... False
273
+ log_world_size_to_tensorboard ................... False
274
+ logging_level ................................... None
275
+ loss_scale ...................................... None
276
+ loss_scale_window ............................... 1000
277
+ lr .............................................. 0.0001
278
+ lr_decay_iters .................................. 8
279
+ lr_decay_samples ................................ None
280
+ lr_decay_style .................................. cosine
281
+ lr_warmup_fraction .............................. None
282
+ lr_warmup_init .................................. 0.0
283
+ lr_warmup_iters ................................. 5
284
+ lr_warmup_samples ............................... 0
285
+ lr_wsd_decay_iters .............................. None
286
+ lr_wsd_decay_samples ............................ None
287
+ lr_wsd_decay_style .............................. exponential
288
+ main_grads_dtype ................................ torch.float32
289
+ main_params_dtype ............................... torch.float32
290
+ make_vocab_size_divisible_by .................... 128
291
+ mamba_head_dim .................................. 64
292
+ mamba_num_groups ................................ 8
293
+ mamba_num_heads ................................. None
294
+ mamba_state_dim ................................. 128
295
+ manual_gc ....................................... False
296
+ manual_gc_eval .................................. True
297
+ manual_gc_interval .............................. 0
298
+ mask_factor ..................................... 1.0
299
+ mask_prob ....................................... 0.15
300
+ mask_type ....................................... random
301
+ masked_softmax_fusion ........................... False
302
+ max_position_embeddings ......................... 32768
303
+ max_tokens_to_oom ............................... 12000
304
+ memory_snapshot_path ............................ snapshot.pickle
305
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
306
+ micro_batch_size ................................ 1
307
+ microbatch_group_size_per_vp_stage .............. None
308
+ mid_level_dataset_surplus ....................... 0.005
309
+ min_loss_scale .................................. 1.0
310
+ min_lr .......................................... 1e-05
311
+ mlp_chunks_for_prefill .......................... 1
312
+ mmap_bin_files .................................. True
313
+ mock_data ....................................... False
314
+ moe_apply_probs_on_input ........................ False
315
+ moe_aux_loss_coeff .............................. 0.01
316
+ moe_capacity_monitor ............................ True
317
+ moe_debug_drop_json ............................. None
318
+ moe_debug_drop_logging .......................... True
319
+ moe_debug_drop_logging_interval ................. 1
320
+ moe_deepep_num_sms .............................. 20
321
+ moe_echo_enable_random_offloading ............... False
322
+ moe_echo_expert_dispatch_overlap ................ False
323
+ moe_echo_expert_dispatcher_type ................. hybridep
324
+ moe_echo_recompute_expert_dispatch .............. False
325
+ moe_enable_deepep ............................... False
326
+ moe_enable_echo ................................. False
327
+ moe_enable_hybridep ............................. True
328
+ moe_enable_ultraep .............................. True
329
+ moe_expert_capacity_factor ...................... None
330
+ moe_extended_tp ................................. False
331
+ moe_ffn_hidden_size ............................. 2560
332
+ moe_grouped_gemm ................................ True
333
+ moe_hybridep_num_sms ............................ 16
334
+ moe_input_jitter_eps ............................ None
335
+ moe_layer_freq .................................. 1
336
+ moe_layer_recompute ............................. False
337
+ moe_num_echo_experts ............................ None
338
+ moe_num_redundant_experts_per_rank .............. 8
339
+ moe_pad_expert_input_to_capacity ................ False
340
+ moe_per_layer_logging ........................... False
341
+ moe_permute_fusion .............................. False
342
+ moe_received_token_capacity ..................... 1.25
343
+ moe_router_bias_update_rate ..................... 0.001
344
+ moe_router_dtype ................................ fp32
345
+ moe_router_enable_expert_bias ................... False
346
+ moe_router_force_load_balancing ................. True
347
+ moe_router_group_topk ........................... None
348
+ moe_router_load_balancing_type .................. aux_loss
349
+ moe_router_num_groups ........................... None
350
+ moe_router_padding_for_fp8 ...................... False
351
+ moe_router_pre_softmax .......................... True
352
+ moe_router_score_function ....................... softmax
353
+ moe_router_topk ................................. 8
354
+ moe_router_topk_scaling_factor .................. None
355
+ moe_shared_expert_intermediate_size ............. 20480
356
+ moe_shared_expert_overlap ....................... False
357
+ moe_token_dispatcher_type ....................... flex
358
+ moe_token_drop_policy ........................... probs
359
+ moe_ultraep_max_microbatches .................... None
360
+ moe_ultraep_overlap_weight_sync ................. True
361
+ moe_ultraep_weight_sync_join .................... before_dispatch
362
+ moe_upcycling_granularity ....................... 1
363
+ moe_use_device_initiated_grouped_gemm ........... True
364
+ moe_use_legacy_grouped_gemm ..................... False
365
+ moe_use_upcycling ............................... False
366
+ moe_z_loss_coeff ................................ None
367
+ mrope_section ................................... None
368
+ mscale .......................................... 1.0
369
+ mscale_all_dim .................................. 0.0
370
+ mtp_loss_scaling_factor ......................... 0.1
371
+ mtp_num_layers .................................. None
372
+ multi_latent_attention .......................... False
373
+ nccl_all_reduce_for_prefill ..................... False
374
+ nccl_communicator_config_path ................... None
375
+ nccl_ub ......................................... False
376
+ no_load_optim ................................... None
377
+ no_load_rng ..................................... None
378
+ no_persist_layer_norm ........................... False
379
+ no_rope_freq .................................... None
380
+ no_save_optim ................................... None
381
+ no_save_rng ..................................... None
382
+ non_persistent_ckpt_type ........................ None
383
+ non_persistent_global_ckpt_dir .................. None
384
+ non_persistent_local_ckpt_algo .................. fully_parallel
385
+ non_persistent_local_ckpt_dir ................... None
386
+ non_persistent_save_interval .................... None
387
+ norm_epsilon .................................... 1e-06
388
+ normalization ................................... RMSNorm
389
+ num_attention_heads ............................. 28
390
+ num_channels .................................... 3
391
+ num_classes ..................................... 1000
392
+ num_dataset_builder_threads ..................... 1
393
+ num_distributed_optimizer_instances ............. 1
394
+ num_experts ..................................... 64
395
+ num_layers ...................................... 10
396
+ num_layers_at_end_in_bf16 ....................... 1
397
+ num_layers_at_start_in_bf16 ..................... 1
398
+ num_layers_per_virtual_pipeline_stage ........... None
399
+ num_query_groups ................................ 4
400
+ num_virtual_stages_per_pipeline_rank ............ None
401
+ num_workers ..................................... 2
402
+ object_storage_cache_path ....................... None
403
+ one_logger_async ................................ False
404
+ one_logger_project .............................. megatron-lm
405
+ one_logger_run_name ............................. None
406
+ onnx_safe ....................................... None
407
+ openai_gelu ..................................... False
408
+ optimizer ....................................... adam
409
+ optimizer_cpu_offload ........................... False
410
+ optimizer_offload_fraction ...................... 1.0
411
+ output_bert_embeddings .......................... False
412
+ overlap_cpu_optimizer_d2h_h2d ................... False
413
+ overlap_grad_reduce ............................. True
414
+ overlap_moe_expert_parallel_comm ................ False
415
+ overlap_p2p_comm ................................ False
416
+ overlap_p2p_comm_warmup_flush ................... False
417
+ overlap_param_gather ............................ True
418
+ overlap_param_gather_with_optimizer_step ........ False
419
+ override_opt_param_scheduler .................... False
420
+ params_dtype .................................... torch.bfloat16
421
+ patch_dim ....................................... 16
422
+ per_split_data_args_path ........................ None
423
+ perform_initialization .......................... True
424
+ pin_cpu_grads ................................... True
425
+ pin_cpu_params .................................. True
426
+ pipeline_model_parallel_comm_backend ............ None
427
+ pipeline_model_parallel_layout .................. None
428
+ pipeline_model_parallel_size .................... 1
429
+ position_embedding_type ......................... rope
430
+ pretrained_checkpoint ........................... None
431
+ profile ......................................... False
432
+ profile_ranks ................................... [0]
433
+ profile_step_end ................................ 12
434
+ profile_step_start .............................. 10
435
+ q_lora_rank ..................................... None
436
+ qk_head_dim ..................................... 128
437
+ qk_l2_norm ...................................... False
438
+ qk_layernorm .................................... False
439
+ qk_pos_emb_head_dim ............................. 64
440
+ query_in_block_prob ............................. 0.1
441
+ rampup_batch_size ............................... None
442
+ rank ............................................ 0
443
+ recompute_granularity ........................... None
444
+ recompute_method ................................ None
445
+ recompute_modules ............................... None
446
+ recompute_num_layers ............................ None
447
+ record_memory_history ........................... False
448
+ relative_attention_max_distance ................. 128
449
+ relative_attention_num_buckets .................. 32
450
+ replication ..................................... False
451
+ replication_factor .............................. 2
452
+ replication_jump ................................ None
453
+ rerun_mode ...................................... validate_results
454
+ reset_attention_mask ............................ False
455
+ reset_position_ids .............................. False
456
+ result_rejected_tracker_filename ................ None
457
+ retriever_report_topk_accuracies ................ []
458
+ retriever_score_scaling ......................... False
459
+ retriever_seq_length ............................ 256
460
+ retro_add_retriever ............................. False
461
+ retro_attention_gate ............................ 1
462
+ retro_cyclic_train_iters ........................ None
463
+ retro_encoder_attention_dropout ................. 0.1
464
+ retro_encoder_hidden_dropout .................... 0.1
465
+ retro_encoder_layers ............................ 2
466
+ retro_num_neighbors ............................. 2
467
+ retro_num_retrieved_chunks ...................... 2
468
+ retro_project_dir ............................... None
469
+ retro_verify_neighbor_count ..................... True
470
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
471
+ rope_scaling_factor ............................. 8.0
472
+ rope_type ....................................... None
473
+ rotary_base ..................................... 1000000
474
+ rotary_interleaved .............................. False
475
+ rotary_percent .................................. 1.0
476
+ rotary_scaling_factor ........................... 1.0
477
+ rotary_seq_len_interpolation_factor ............. None
478
+ run_workload_inspector_server ................... False
479
+ sample_rate ..................................... 1.0
480
+ save ............................................ None
481
+ save_interval ................................... None
482
+ save_retain_interval ............................ None
483
+ scatter_gather_tensors_in_pipeline .............. True
484
+ seed ............................................ 1234
485
+ seq_length ...................................... 4096
486
+ sequence_parallel ............................... False
487
+ sft ............................................. False
488
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
489
+ sgd_momentum .................................... 0.9
490
+ short_seq_prob .................................. 0.1
491
+ skip_train ...................................... False
492
+ skipped_train_samples ........................... 0
493
+ spec ............................................ None
494
+ split ........................................... 100,0,0
495
+ squared_relu .................................... False
496
+ start_weight_decay .............................. 0.1
497
+ straggler_ctrlr_port ............................ 65535
498
+ straggler_minmax_count .......................... 1
499
+ suggested_communication_unit_size ............... None
500
+ swiglu .......................................... True
501
+ swin_backbone_type .............................. tiny
502
+ symmetric_ar_type ............................... None
503
+ te_rng_tracker .................................. False
504
+ tensor_model_parallel_size ...................... 1
505
+ tensorboard_dir ................................. None
506
+ tensorboard_log_interval ........................ 1
507
+ tensorboard_queue_size .......................... 1000
508
+ test_data_path .................................. None
509
+ test_mode ....................................... False
510
+ tiktoken_num_special_tokens ..................... 1000
511
+ tiktoken_pattern ................................ None
512
+ tiktoken_special_tokens ......................... None
513
+ timing_log_level ................................ 0
514
+ timing_log_option ............................... minmax
515
+ titles_data_path ................................ None
516
+ tokenizer_model ................................. None
517
+ tokenizer_type .................................. GPT2BPETokenizer
518
+ torch_fsdp2_reshard_after_forward ............... True
519
+ tp_comm_bootstrap_backend ....................... nccl
520
+ tp_comm_bulk_dgrad .............................. True
521
+ tp_comm_bulk_wgrad .............................. True
522
+ tp_comm_overlap ................................. False
523
+ tp_comm_overlap_ag .............................. True
524
+ tp_comm_overlap_cfg ............................. None
525
+ tp_comm_overlap_rs .............................. True
526
+ tp_comm_overlap_rs_dgrad ........................ False
527
+ tp_comm_split_ag ................................ True
528
+ tp_comm_split_rs ................................ True
529
+ train_data_path ................................. None
530
+ train_iters ..................................... 12
531
+ train_samples ................................... None
532
+ train_sync_interval ............................. None
533
+ transformer_impl ................................ transformer_engine
534
+ transformer_pipeline_model_parallel_size ........ 1
535
+ untie_embeddings_and_output_weights ............. True
536
+ use_checkpoint_args ............................. False
537
+ use_checkpoint_opt_param_scheduler .............. False
538
+ use_cpu_initialization .......................... None
539
+ use_custom_fsdp ................................. False
540
+ use_dist_ckpt ................................... True
541
+ use_dist_ckpt_deprecated ........................ False
542
+ use_distributed_optimizer ....................... True
543
+ use_flash_attn .................................. False
544
+ use_legacy_models ............................... False
545
+ use_mp_args_from_checkpoint_args ................ False
546
+ use_one_sent_docs ............................... False
547
+ use_persistent_ckpt_worker ...................... False
548
+ use_precision_aware_optimizer ................... False
549
+ use_pytorch_profiler ............................ False
550
+ use_ring_exchange_p2p ........................... False
551
+ use_rope_scaling ................................ False
552
+ use_rotary_position_embeddings .................. False
553
+ use_sharp ....................................... False
554
+ use_tokenizer_model_from_checkpoint_args ........ True
555
+ use_torch_fsdp2 ................................. False
556
+ use_torch_optimizer_for_cpu_offload ............. False
557
+ use_tp_pp_dp_mapping ............................ False
558
+ v_head_dim ...................................... 128
559
+ valid_data_path ................................. None
560
+ variable_seq_lengths ............................ False
561
+ virtual_pipeline_model_parallel_size ............ None
562
+ vision_backbone_type ............................ vit
563
+ vision_pretraining .............................. False
564
+ vision_pretraining_type ......................... classify
565
+ vocab_extra_ids ................................. 0
566
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
567
+ vocab_size ...................................... None
568
+ wandb_exp_name ..................................
569
+ wandb_project ...................................
570
+ wandb_save_dir ..................................
571
+ weight_decay .................................... 0.1
572
+ weight_decay_incr_style ......................... constant
573
+ wgrad_deferral_limit ............................ 0
574
+ world_size ...................................... 8
575
+ yaml_cfg ........................................ None
576
+ -------------------- end of arguments ---------------------
577
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
578
+ > building GPT2BPETokenizer tokenizer ...
579
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
580
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
581
+ > initializing torch distributed ...
582
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
583
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
584
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
585
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
586
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank [Gloo] Rank 4 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
588
+ 7
589
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
598
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
599
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
600
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
601
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
602
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
603
+ > initialized tensor model parallel with size 1
604
+ > initialized pipeline model parallel with size 1
605
+ > setting random seeds to 1234 ...
606
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
607
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
610
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
611
+ [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
612
+
613
+ [rank3]:[W826 15:52:36.064778057 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank7]:[W826 15:52:36.064811881 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank2]:[W826 15:52:36.064811429 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
+ [rank1]:[W826 15:52:36.064844464 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
617
+ [rank5]:[W826 15:52:36.064870774 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
618
+ [rank6]:[W826 15:52:36.064952671 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
619
+ [rank4]:[W826 15:52:36.065191833 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
620
+ > compiling dataset index builder ...
621
+ make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
622
+ make: Nothing to be done for 'default'.
623
+ make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
624
+ >>> done with dataset index builder. Compilation time: 0.033 seconds
625
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
626
+ > compiling and loading fused kernels ...
627
+ [rank0]:[W826 15:52:36.219780669 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
628
+ >>> done with compiling and loading fused kernels. Compilation time: 3.961 seconds
629
+ time to initialize megatron (seconds): 6.921
630
+ [after megatron is initialized] datetime: 2026-08-26 15:52:42
631
+ building GPT model ...
632
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
633
+ warnings.warn(
634
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
635
+ warnings.warn(
636
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
637
+ warnings.warn(
638
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
639
+ warnings.warn(
640
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
641
+ warnings.warn(
642
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
643
+ warnings.warn(
644
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
645
+ warnings.warn(
646
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
647
+ warnings.warn(
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 1
18
  1, 1
19
  2, 1
20
  3, 1
@@ -25,7 +25,7 @@
25
  0, 4
26
  1, 4
27
  2, 4
28
- 3, 1
29
  4, 1
30
  5, 1
31
  6, 1
@@ -44,88 +44,56 @@
44
  3, 4
45
  4, 4
46
  5, 4
47
- 6, 4
48
  7, 4
49
- 0, 4
50
- 1, 4
51
- 2, 4
52
- 3, 629
53
- 4, 4
54
- 5, 629
55
- 6, 629
56
- 7, 4
57
- 0, 725
58
- 1, 725
59
- 2, 725
60
- 3, 725
61
- 4, 725
62
- 5, 725
63
- 6, 725
64
- 7, 725
65
- 0, 725
66
- 1, 725
67
- 2, 725
68
- 3, 725
69
- 4, 725
70
- 5, 725
71
- 6, 725
72
- 7, 725
73
  0, 821
74
- 1, 819
75
- 2, 819
76
  3, 821
77
- 4, 819
78
  5, 821
79
- 6, 819
80
- 7, 819
81
- 0, 1537
82
- 1, 1537
83
- 2, 1537
84
- 3, 1537
85
- 4, 1537
86
- 5, 1537
87
- 6, 1537
88
- 7, 1537
89
- 0, 2551
90
- 1, 2551
91
- 2, 2551
92
- 3, 2551
93
- 4, 2551
94
- 5, 2551
95
- 6, 2551
96
- 7, 2551
97
- 0, 2959
98
- 1, 2959
99
- 2, 2959
100
- 3, 2959
101
- 4, 2959
102
- 5, 2959
103
- 6, 2959
104
- 7, 2959
105
- 0, 51537
106
- 1, 51537
107
- 2, 51537
108
- 3, 51537
109
- 4, 51249
110
- 5, 51537
111
- 6, 51537
112
- 7, 51537
113
- 0, 52789
114
- 1, 52789
115
- 2, 52789
116
- 3, 52789
117
- 4, 52501
118
- 5, 52789
119
- 6, 52791
120
- 7, 52789
121
- 0, 53651
122
- 1, 53631
123
- 2, 53647
124
- 3, 53643
125
- 4, 53349
126
- 5, 53629
127
- 6, 53633
128
- 7, 53625
129
  0, 54649
130
  1, 54649
131
  2, 54649
@@ -150,22 +118,22 @@
150
  5, 57201
151
  6, 57201
152
  7, 57201
153
- 0, 57201
154
- 1, 57201
155
- 2, 57201
156
- 3, 57201
157
- 4, 56913
158
- 5, 57201
159
- 6, 57201
160
- 7, 57201
161
- 0, 90609
162
- 1, 90671
163
- 2, 90681
164
- 3, 90681
165
- 4, 90613
166
- 5, 90667
167
- 6, 90665
168
- 7, 90647
169
  0, 91083
170
  1, 91131
171
  2, 91131
@@ -182,30 +150,30 @@
182
  5, 91131
183
  6, 91131
184
  7, 91131
185
- 0, 91083
186
- 1, 91131
187
- 2, 91131
188
- 3, 91131
189
- 4, 91125
190
- 5, 91131
191
- 6, 91131
192
- 7, 91131
193
- 0, 91375
194
- 1, 91441
195
- 2, 91463
196
- 3, 91479
197
- 4, 91453
198
- 5, 91427
199
- 6, 91417
200
- 7, 91429
201
- 0, 95881
202
- 1, 95959
203
- 2, 95935
204
- 3, 95965
205
- 4, 96031
206
- 5, 95921
207
- 6, 95991
208
- 7, 95941
209
  0, 122617
210
  1, 122751
211
  2, 122751
@@ -222,19 +190,35 @@
222
  5, 122751
223
  6, 122751
224
  7, 122751
225
- 0, 122617
226
- 1, 122751
227
- 2, 122751
228
- 3, 122751
229
- 4, 122655
230
- 5, 122751
231
- 6, 122751
232
- 7, 122751
233
- 0, 5277
234
- 1, 3678
235
- 2, 3678
236
- 3, 3678
237
- 4, 3678
238
- 5, 3678
239
- 6, 3998
240
- 7, 122749
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 4
18
  1, 1
19
  2, 1
20
  3, 1
 
25
  0, 4
26
  1, 4
27
  2, 4
28
+ 3, 4
29
  4, 1
30
  5, 1
31
  6, 1
 
44
  3, 4
45
  4, 4
46
  5, 4
47
+ 6, 47
48
  7, 4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  0, 821
50
+ 1, 821
51
+ 2, 821
52
  3, 821
53
+ 4, 821
54
  5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1975
58
+ 1, 1975
59
+ 2, 1975
60
+ 3, 1975
61
+ 4, 1975
62
+ 5, 1975
63
+ 6, 1975
64
+ 7, 1975
65
+ 0, 2417
66
+ 1, 2419
67
+ 2, 2421
68
+ 3, 2403
69
+ 4, 2407
70
+ 5, 2407
71
+ 6, 2431
72
+ 7, 2437
73
+ 0, 22733
74
+ 1, 15429
75
+ 2, 15431
76
+ 3, 15431
77
+ 4, 15427
78
+ 5, 15427
79
+ 6, 29645
80
+ 7, 15033
81
+ 0, 51917
82
+ 1, 51895
83
+ 2, 51937
84
+ 3, 51919
85
+ 4, 51703
86
+ 5, 51925
87
+ 6, 51917
88
+ 7, 51935
89
+ 0, 53635
90
+ 1, 53601
91
+ 2, 53605
92
+ 3, 53581
93
+ 4, 53301
94
+ 5, 53585
95
+ 6, 53595
96
+ 7, 53583
 
 
 
 
 
 
 
 
97
  0, 54649
98
  1, 54649
99
  2, 54649
 
118
  5, 57201
119
  6, 57201
120
  7, 57201
121
+ 0, 60853
122
+ 1, 60853
123
+ 2, 60853
124
+ 3, 60853
125
+ 4, 60201
126
+ 5, 60853
127
+ 6, 60853
128
+ 7, 60853
129
+ 0, 90695
130
+ 1, 90743
131
+ 2, 90743
132
+ 3, 90743
133
+ 4, 90727
134
+ 5, 90743
135
+ 6, 90743
136
+ 7, 90743
137
  0, 91083
138
  1, 91131
139
  2, 91131
 
150
  5, 91131
151
  6, 91131
152
  7, 91131
153
+ 0, 91331
154
+ 1, 91465
155
+ 2, 91397
156
+ 3, 91475
157
+ 4, 91355
158
+ 5, 91395
159
+ 6, 91389
160
+ 7, 91397
161
+ 0, 95905
162
+ 1, 96031
163
+ 2, 95969
164
+ 3, 96045
165
+ 4, 96083
166
+ 5, 95949
167
+ 6, 95975
168
+ 7, 95961
169
+ 0, 109105
170
+ 1, 109153
171
+ 2, 111233
172
+ 3, 109153
173
+ 4, 109265
174
+ 5, 109153
175
+ 6, 116783
176
+ 7, 109153
177
  0, 122617
178
  1, 122751
179
  2, 122751
 
190
  5, 122751
191
  6, 122751
192
  7, 122751
193
+ 0, 118287
194
+ 1, 118421
195
+ 2, 118421
196
+ 3, 118421
197
+ 4, 118325
198
+ 5, 118421
199
+ 6, 118421
200
+ 7, 118421
201
+ 0, 116217
202
+ 1, 116741
203
+ 2, 116635
204
+ 3, 116741
205
+ 4, 116257
206
+ 5, 116741
207
+ 6, 116353
208
+ 7, 116511
209
+ 0, 4
210
+ 1, 114673
211
+ 2, 114673
212
+ 3, 115061
213
+ 4, 114577
214
+ 5, 114673
215
+ 6, 114673
216
+ 7, 114673
217
+ 0, 1
218
+ 1, 4
219
+ 2, 4
220
+ 3, 114671
221
+ 4, 4
222
+ 5, 4
223
+ 6, 4
224
+ 7, 4
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 2
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 165
43
+ 2, 629
44
+ 3, 629
45
+ 4, 165
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 1045
50
+ 1, 1043
51
+ 2, 1069
52
+ 3, 1065
53
+ 4, 1049
54
+ 5, 1057
55
+ 6, 1033
56
+ 7, 1031
57
+ 0, 1975
58
+ 1, 1975
59
+ 2, 1975
60
+ 3, 1975
61
+ 4, 1975
62
+ 5, 1975
63
+ 6, 1975
64
+ 7, 1975
65
+ 0, 2571
66
+ 1, 2571
67
+ 2, 2571
68
+ 3, 2571
69
+ 4, 2563
70
+ 5, 2549
71
+ 6, 2541
72
+ 7, 2569
73
+ 0, 43469
74
+ 1, 43469
75
+ 2, 22733
76
+ 3, 22733
77
+ 4, 43469
78
+ 5, 43469
79
+ 6, 43469
80
+ 7, 43469
81
+ 0, 52055
82
+ 1, 52055
83
+ 2, 52055
84
+ 3, 52055
85
+ 4, 51767
86
+ 5, 52055
87
+ 6, 52055
88
+ 7, 52055
89
+ 0, 53661
90
+ 1, 53661
91
+ 2, 53661
92
+ 3, 53661
93
+ 4, 53373
94
+ 5, 53661
95
+ 6, 53661
96
+ 7, 53661
97
+ 0, 54649
98
+ 1, 54649
99
+ 2, 54649
100
+ 3, 54649
101
+ 4, 54361
102
+ 5, 54649
103
+ 6, 54649
104
+ 7, 54649
105
+ 0, 55289
106
+ 1, 55289
107
+ 2, 55289
108
+ 3, 55289
109
+ 4, 55001
110
+ 5, 55289
111
+ 6, 55289
112
+ 7, 55289
113
+ 0, 57201
114
+ 1, 57201
115
+ 2, 57201
116
+ 3, 57201
117
+ 4, 56913
118
+ 5, 57201
119
+ 6, 57201
120
+ 7, 57201
121
+ 0, 68039
122
+ 1, 68195
123
+ 2, 68145
124
+ 3, 68675
125
+ 4, 68067
126
+ 5, 68183
127
+ 6, 68121
128
+ 7, 68183
129
+ 0, 90699
130
+ 1, 90755
131
+ 2, 90755
132
+ 3, 90755
133
+ 4, 90739
134
+ 5, 90755
135
+ 6, 90755
136
+ 7, 90755
137
+ 0, 91083
138
+ 1, 91139
139
+ 2, 91139
140
+ 3, 91139
141
+ 4, 91123
142
+ 5, 91139
143
+ 6, 91139
144
+ 7, 91139
145
+ 0, 91083
146
+ 1, 91139
147
+ 2, 91139
148
+ 3, 91139
149
+ 4, 91125
150
+ 5, 91139
151
+ 6, 91139
152
+ 7, 91139
153
+ 0, 91325
154
+ 1, 91379
155
+ 2, 91423
156
+ 3, 91459
157
+ 4, 91373
158
+ 5, 91437
159
+ 6, 91449
160
+ 7, 91371
161
+ 0, 95859
162
+ 1, 95925
163
+ 2, 95945
164
+ 3, 95891
165
+ 4, 96053
166
+ 5, 95901
167
+ 6, 95917
168
+ 7, 95927
169
+ 0, 108009
170
+ 1, 109161
171
+ 2, 109161
172
+ 3, 109161
173
+ 4, 109905
174
+ 5, 109161
175
+ 6, 109161
176
+ 7, 109161
177
+ 0, 122619
178
+ 1, 122739
179
+ 2, 122739
180
+ 3, 122739
181
+ 4, 122655
182
+ 5, 122739
183
+ 6, 122739
184
+ 7, 122739
185
+ 0, 122619
186
+ 1, 122737
187
+ 2, 122739
188
+ 3, 122739
189
+ 4, 122655
190
+ 5, 122739
191
+ 6, 122739
192
+ 7, 122739
193
+ 0, 3675
194
+ 1, 3678
195
+ 2, 3678
196
+ 3, 3678
197
+ 4, 3998
198
+ 5, 122737
199
+ 6, 5278
200
+ 7, 3678
201
+ 0, 0
202
+ 1, 0
203
+ 2, 0
204
+ 3, 0
205
+ 4, 0
206
+ 5, 0
207
+ 6, 0
208
+ 7, 0
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-token-drop-analysis
6
+ commit=eaf829e7
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=512
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv CHANGED
@@ -24,9 +24,17 @@
24
  7, 1
25
  0, 4
26
  1, 4
 
 
 
 
 
 
 
 
27
  2, 4
28
  3, 4
29
- 4, 1
30
  5, 1
31
  6, 1
32
  7, 1
@@ -39,21 +47,21 @@
39
  6, 4
40
  7, 4
41
  0, 4
42
- 1, 629
43
  2, 629
44
- 3, 629
45
- 4, 629
46
- 5, 629
47
- 6, 629
48
- 7, 629
49
- 0, 921
50
- 1, 913
51
- 2, 919
52
- 3, 923
53
- 4, 909
54
- 5, 913
55
- 6, 909
56
- 7, 915
57
  0, 1831
58
  1, 1831
59
  2, 1831
@@ -62,38 +70,38 @@
62
  5, 1831
63
  6, 1831
64
  7, 1831
65
- 0, 2217
66
- 1, 2221
67
- 2, 2219
68
- 3, 2221
69
- 4, 2197
70
- 5, 2215
71
- 6, 2209
72
- 7, 2219
73
- 0, 11431
74
- 1, 11511
75
- 2, 11311
76
- 3, 11811
77
- 4, 11711
78
- 5, 11711
79
- 6, 11511
80
- 7, 11511
81
- 0, 47039
82
- 1, 47017
83
- 2, 47057
84
- 3, 47021
85
- 4, 47047
86
- 5, 47059
87
- 6, 47037
88
- 7, 47051
89
- 0, 47829
90
- 1, 47831
91
- 2, 47841
92
- 3, 47803
93
- 4, 47827
94
- 5, 47841
95
- 6, 47831
96
- 7, 47841
97
  0, 48645
98
  1, 48647
99
  2, 48657
@@ -110,123 +118,123 @@
110
  5, 48817
111
  6, 48807
112
  7, 48817
113
- 0, 49285
114
- 1, 49287
115
- 2, 49293
116
- 3, 49259
117
- 4, 49259
118
- 5, 49293
119
- 6, 49243
120
- 7, 49293
121
- 0, 49577
122
- 1, 49579
123
- 2, 49581
124
- 3, 49551
125
- 4, 49551
126
- 5, 49581
127
- 6, 49535
128
- 7, 49581
129
- 0, 85197
130
- 1, 85985
131
- 2, 85195
132
- 3, 85211
133
- 4, 85211
134
- 5, 85195
135
- 6, 85195
136
- 7, 85195
137
- 0, 86697
138
- 1, 86699
139
- 2, 86695
140
- 3, 86711
141
- 4, 86711
142
- 5, 86695
143
- 6, 86695
144
- 7, 86695
145
- 0, 86697
146
- 1, 86699
147
- 2, 86695
148
- 3, 86711
149
- 4, 86711
150
- 5, 86695
151
- 6, 86695
152
- 7, 86695
153
- 0, 86697
154
- 1, 86699
155
- 2, 86695
156
- 3, 86711
157
- 4, 86711
158
- 5, 86695
159
- 6, 86695
160
- 7, 86695
161
- 0, 86709
162
- 1, 86711
163
  2, 86707
164
- 3, 86723
165
- 4, 86723
166
  5, 86707
167
- 6, 86707
168
  7, 86707
169
- 0, 87163
170
- 1, 87177
171
- 2, 87177
172
- 3, 87209
173
- 4, 87189
174
- 5, 87179
175
- 6, 87181
176
- 7, 87167
177
- 0, 88409
178
- 1, 88341
179
- 2, 88371
180
- 3, 88373
181
- 4, 88387
182
- 5, 88351
183
- 6, 88361
184
- 7, 88341
185
- 0, 117865
186
- 1, 117863
187
- 2, 117903
188
- 3, 117859
189
- 4, 117915
190
- 5, 117903
191
- 6, 117863
192
- 7, 117903
193
- 0, 117865
194
- 1, 117863
195
- 2, 117903
196
- 3, 117859
197
- 4, 117915
198
- 5, 117903
199
- 6, 117863
200
- 7, 117903
201
- 0, 117865
202
- 1, 117863
203
- 2, 117903
204
- 3, 117859
205
- 4, 117915
206
- 5, 117903
207
- 6, 117863
208
- 7, 117903
209
- 0, 117865
210
- 1, 117863
211
- 2, 117903
212
- 3, 117859
213
- 4, 117915
214
- 5, 117903
215
- 6, 117863
216
- 7, 117903
217
- 0, 117865
218
- 1, 117863
219
- 2, 117903
220
- 3, 117859
221
- 4, 117915
222
- 5, 117903
223
- 6, 117863
224
- 7, 117903
225
- 0, 1709
226
- 1, 1712
227
- 2, 2032
228
- 3, 117859
229
- 4, 3312
230
- 5, 1712
231
- 6, 1712
232
- 7, 1712
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  7, 1
25
  0, 4
26
  1, 4
27
+ 2, 1
28
+ 3, 1
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
  2, 4
36
  3, 4
37
+ 4, 2
38
  5, 1
39
  6, 1
40
  7, 1
 
47
  6, 4
48
  7, 4
49
  0, 4
50
+ 1, 4
51
  2, 629
52
+ 3, 4
53
+ 4, 4
54
+ 5, 4
55
+ 6, 4
56
+ 7, 47
57
+ 0, 821
58
+ 1, 821
59
+ 2, 821
60
+ 3, 821
61
+ 4, 821
62
+ 5, 821
63
+ 6, 821
64
+ 7, 821
65
  0, 1831
66
  1, 1831
67
  2, 1831
 
70
  5, 1831
71
  6, 1831
72
  7, 1831
73
+ 0, 2175
74
+ 1, 2163
75
+ 2, 2161
76
+ 3, 2185
77
+ 4, 2165
78
+ 5, 2185
79
+ 6, 2175
80
+ 7, 2175
81
+ 0, 10651
82
+ 1, 10631
83
+ 2, 10431
84
+ 3, 10431
85
+ 4, 10831
86
+ 5, 10431
87
+ 6, 10431
88
+ 7, 10431
89
+ 0, 46993
90
+ 1, 47007
91
+ 2, 47001
92
+ 3, 46955
93
+ 4, 46985
94
+ 5, 47037
95
+ 6, 47019
96
+ 7, 47007
97
+ 0, 48475
98
+ 1, 48477
99
+ 2, 48487
100
+ 3, 48449
101
+ 4, 48473
102
+ 5, 48487
103
+ 6, 48477
104
+ 7, 48487
105
  0, 48645
106
  1, 48647
107
  2, 48657
 
118
  5, 48817
119
  6, 48807
120
  7, 48817
121
+ 0, 49509
122
+ 1, 49511
123
+ 2, 49513
124
+ 3, 49483
125
+ 4, 49483
126
+ 5, 49513
127
+ 6, 49467
128
+ 7, 49513
129
+ 0, 60359
130
+ 1, 60381
131
+ 2, 60377
132
+ 3, 60353
133
+ 4, 60371
134
+ 5, 60357
135
+ 6, 60337
136
+ 7, 60419
137
+ 0, 86153
138
+ 1, 86111
139
+ 2, 86147
140
+ 3, 86135
141
+ 4, 86117
142
+ 5, 86153
143
+ 6, 86125
144
+ 7, 86139
145
+ 0, 86689
146
+ 1, 86691
147
+ 2, 86707
148
+ 3, 86703
149
+ 4, 86703
150
+ 5, 86707
151
+ 6, 86687
152
+ 7, 86707
153
+ 0, 86689
154
+ 1, 86691
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
  2, 86707
156
+ 3, 86703
157
+ 4, 86703
158
  5, 86707
159
+ 6, 86687
160
  7, 86707
161
+ 0, 86955
162
+ 1, 86959
163
+ 2, 86961
164
+ 3, 87009
165
+ 4, 86979
166
+ 5, 86993
167
+ 6, 87027
168
+ 7, 87035
169
+ 0, 87999
170
+ 1, 87985
171
+ 2, 87997
172
+ 3, 87995
173
+ 4, 87955
174
+ 5, 88035
175
+ 6, 87957
176
+ 7, 87983
177
+ 0, 107875
178
+ 1, 108077
179
+ 2, 106683
180
+ 3, 107349
181
+ 4, 107589
182
+ 5, 108123
183
+ 6, 107893
184
+ 7, 106763
185
+ 0, 117877
186
+ 1, 117855
187
+ 2, 117885
188
+ 3, 117871
189
+ 4, 117927
190
+ 5, 117885
191
+ 6, 117855
192
+ 7, 117885
193
+ 0, 117877
194
+ 1, 117855
195
+ 2, 117885
196
+ 3, 117871
197
+ 4, 117927
198
+ 5, 117885
199
+ 6, 117855
200
+ 7, 117885
201
+ 0, 117877
202
+ 1, 117855
203
+ 2, 117885
204
+ 3, 117871
205
+ 4, 117927
206
+ 5, 117885
207
+ 6, 117855
208
+ 7, 117885
209
+ 0, 117877
210
+ 1, 117855
211
+ 2, 117885
212
+ 3, 117871
213
+ 4, 117927
214
+ 5, 117885
215
+ 6, 117855
216
+ 7, 117885
217
+ 0, 115519
218
+ 1, 115497
219
+ 2, 115527
220
+ 3, 115513
221
+ 4, 115569
222
+ 5, 115527
223
+ 6, 115497
224
+ 7, 115527
225
+ 0, 113839
226
+ 1, 113817
227
+ 2, 113847
228
+ 3, 113833
229
+ 4, 113889
230
+ 5, 113847
231
+ 6, 113817
232
+ 7, 113847
233
+ 0, 1
234
+ 1, 4
235
+ 2, 4
236
+ 3, 4
237
+ 4, 4
238
+ 5, 132
239
+ 6, 113817
240
+ 7, 644
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,240 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 1
28
+ 3, 1
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 1
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 725
50
+ 1, 725
51
+ 2, 725
52
+ 3, 725
53
+ 4, 725
54
+ 5, 725
55
+ 6, 725
56
+ 7, 725
57
+ 0, 993
58
+ 1, 1015
59
+ 2, 1027
60
+ 3, 1025
61
+ 4, 1003
62
+ 5, 1021
63
+ 6, 1013
64
+ 7, 989
65
+ 0, 1831
66
+ 1, 1831
67
+ 2, 1831
68
+ 3, 1831
69
+ 4, 1831
70
+ 5, 1831
71
+ 6, 1831
72
+ 7, 1831
73
+ 0, 2227
74
+ 1, 2241
75
+ 2, 2227
76
+ 3, 2209
77
+ 4, 2239
78
+ 5, 2205
79
+ 6, 2245
80
+ 7, 2223
81
+ 0, 9851
82
+ 1, 10051
83
+ 2, 10051
84
+ 3, 9951
85
+ 4, 10091
86
+ 5, 10091
87
+ 6, 10091
88
+ 7, 9851
89
+ 0, 47013
90
+ 1, 47079
91
+ 2, 47037
92
+ 3, 46993
93
+ 4, 47039
94
+ 5, 47075
95
+ 6, 47021
96
+ 7, 47061
97
+ 0, 48087
98
+ 1, 48089
99
+ 2, 48099
100
+ 3, 48061
101
+ 4, 48085
102
+ 5, 48099
103
+ 6, 48089
104
+ 7, 48099
105
+ 0, 48645
106
+ 1, 48647
107
+ 2, 48657
108
+ 3, 48619
109
+ 4, 48643
110
+ 5, 48657
111
+ 6, 48647
112
+ 7, 48657
113
+ 0, 48805
114
+ 1, 48807
115
+ 2, 48817
116
+ 3, 48779
117
+ 4, 48803
118
+ 5, 48817
119
+ 6, 48807
120
+ 7, 48817
121
+ 0, 49509
122
+ 1, 49511
123
+ 2, 49513
124
+ 3, 49483
125
+ 4, 49483
126
+ 5, 49513
127
+ 6, 49467
128
+ 7, 49513
129
+ 0, 58785
130
+ 1, 58705
131
+ 2, 58623
132
+ 3, 58719
133
+ 4, 58819
134
+ 5, 58805
135
+ 6, 58783
136
+ 7, 58803
137
+ 0, 86183
138
+ 1, 86191
139
+ 2, 86257
140
+ 3, 86203
141
+ 4, 86225
142
+ 5, 86199
143
+ 6, 86211
144
+ 7, 86253
145
+ 0, 86669
146
+ 1, 86671
147
+ 2, 86687
148
+ 3, 86683
149
+ 4, 86683
150
+ 5, 86687
151
+ 6, 86667
152
+ 7, 86687
153
+ 0, 86669
154
+ 1, 86671
155
+ 2, 86687
156
+ 3, 86683
157
+ 4, 86683
158
+ 5, 86687
159
+ 6, 86667
160
+ 7, 86687
161
+ 0, 87007
162
+ 1, 86955
163
+ 2, 87013
164
+ 3, 87025
165
+ 4, 86981
166
+ 5, 86971
167
+ 6, 86935
168
+ 7, 86975
169
+ 0, 87953
170
+ 1, 87959
171
+ 2, 87983
172
+ 3, 87931
173
+ 4, 87953
174
+ 5, 87937
175
+ 6, 87959
176
+ 7, 87991
177
+ 0, 104475
178
+ 1, 104467
179
+ 2, 104513
180
+ 3, 103749
181
+ 4, 105069
182
+ 5, 105053
183
+ 6, 104893
184
+ 7, 105233
185
+ 0, 117837
186
+ 1, 117825
187
+ 2, 117855
188
+ 3, 117831
189
+ 4, 117867
190
+ 5, 117855
191
+ 6, 117815
192
+ 7, 117855
193
+ 0, 117837
194
+ 1, 117825
195
+ 2, 117855
196
+ 3, 117831
197
+ 4, 117867
198
+ 5, 117855
199
+ 6, 117815
200
+ 7, 117855
201
+ 0, 117837
202
+ 1, 117825
203
+ 2, 117855
204
+ 3, 117831
205
+ 4, 117867
206
+ 5, 117855
207
+ 6, 117815
208
+ 7, 117855
209
+ 0, 117837
210
+ 1, 117825
211
+ 2, 117855
212
+ 3, 117831
213
+ 4, 117867
214
+ 5, 117855
215
+ 6, 117815
216
+ 7, 117855
217
+ 0, 115479
218
+ 1, 115467
219
+ 2, 115497
220
+ 3, 115473
221
+ 4, 115509
222
+ 5, 115497
223
+ 6, 115457
224
+ 7, 115497
225
+ 0, 114251
226
+ 1, 114433
227
+ 2, 114201
228
+ 3, 114439
229
+ 4, 114601
230
+ 5, 114463
231
+ 6, 114097
232
+ 7, 114205
233
+ 0, 1
234
+ 1, 4
235
+ 2, 4
236
+ 3, 132
237
+ 4, 114217
238
+ 5, 644
239
+ 6, 4
240
+ 7, 4
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-token-drop-analysis
6
+ commit=eaf829e7
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=256
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv CHANGED
@@ -1,8 +1,8 @@
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
- QWEN3_235B ultraep measure 1.25 OK 122751 1 0.0 1.01004 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1
3
- QWEN3_235B ultraep validate 1.02 ASSERT 54649 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1
4
- QWEN3_30B ultraep measure 1.25 OK 117915 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
5
- QWEN3_30B ultraep validate 1.02 ASSERT 48657 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1
6
  DSV2_LITE_L24 ultraep measure 1.25 OK 130023 1 0.0 1.010071 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1
7
- DSV2_LITE_L24 ultraep validate 1.02 ASSERT 39855 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1
8
- QWEN2_57B ultraep measure 1.25 FAIL 47627 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
 
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
+ QWEN3_235B ultraep measure 1.25 OK 122751 1 0.0 1.010056 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1
3
+ QWEN3_235B ultraep forced 1.25 OK 122739 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1
4
+ QWEN3_30B ultraep measure 1.25 OK 117927 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
5
+ QWEN3_30B ultraep forced 1.25 OK 117867 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1
6
  DSV2_LITE_L24 ultraep measure 1.25 OK 130023 1 0.0 1.010071 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1
7
+ DSV2_LITE_L24 ultraep forced 1.25 OK 129971 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1
8
+ QWEN2_57B ultraep measure 1.25 OK 97675 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv CHANGED
@@ -1,5 +1,4 @@
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
- QWEN3_235B ultraep ERROR_ASSERT
3
- QWEN3_30B ultraep ERROR_ASSERT
4
- DSV2_LITE_L24 ultraep ERROR_ASSERT
5
- QWEN2_57B ultraep ERROR_FAIL
 
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
+ QWEN3_235B ultraep SELECTED 1.25 1.010056 0.0 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1
3
+ QWEN3_30B ultraep SELECTED 1.25 1.010101 0.0 117927 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
4
+ DSV2_LITE_L24 ultraep SELECTED 1.25 1.010071 0.0 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1
 
tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv CHANGED
@@ -1,8 +1,8 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
- QWEN3_235B ultraep matched 16 8192 8 1.25 none 1 OK 339.56 338.10 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1
3
- QWEN3_235B ultraep matched 16 8192 8 1.02 none 1 ASSERT NA NA 54649 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1
4
- QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 780.66 722.00 117915 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
5
- QWEN3_30B ultraep matched 16 4096 8 1.02 none 1 ASSERT NA NA 48657 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1
6
- DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none 1 OK 641.99 639.30 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1
7
- DSV2_LITE_L24 ultraep matched 8 8192 8 1.02 none 1 ASSERT NA NA 39855 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1
8
- QWEN2_57B ultraep matched 8 4096 8 1.25 none 1 FAIL NA NA 47627 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ QWEN3_235B ultraep matched 16 8192 8 1.25 none 1 OK 340.64 339.20 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1
3
+ QWEN3_235B ultraep matched 16 8192 8 1.25 none+forcebal 1 OK 317.89 317.40 122739 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1
4
+ QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 749.31 712.40 117927 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1
5
+ QWEN3_30B ultraep matched 16 4096 8 1.25 none+forcebal 1 OK 737.56 705.60 117867 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1
6
+ DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none 1 OK 639.96 638.80 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1
7
+ DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none+forcebal 1 OK 625.83 624.60 129971 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1
8
+ QWEN2_57B ultraep matched 8 4096 8 1.25 none 1 OK 238.29 230.10 97675 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1