AbhishekGhosh1998 commited on
Commit
5e7fd80
·
verified ·
1 Parent(s): d22deb8

granularity results granularity_next_b200/tiera_topk_trace @ 2026-09-21T05:52:31Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
Files changed (18) hide show
  1. .gitattributes +4 -0
  2. granularity_next_b200/tiera_topk_trace/hf_watcher.log +3 -0
  3. granularity_next_b200/tiera_topk_trace/study.log +23 -0
  4. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1.nsys-rep +3 -0
  5. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/run.env +102 -0
  6. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/train.log +0 -0
  7. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/verify_trace_ranks.log +6 -0
  8. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125.nsys-rep +3 -0
  9. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/run.env +102 -0
  10. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/train.log +0 -0
  11. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep +3 -0
  12. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/train.log +49 -0
  13. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/verify_trace_ranks.log +6 -0
  14. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375.nsys-rep +3 -0
  15. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/run.env +102 -0
  16. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/train.log +0 -0
  17. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/verify_trace_ranks.log +6 -0
  18. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/trace_results.tsv +3 -0
.gitattributes CHANGED
@@ -96,3 +96,7 @@ granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K2
96
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep filter=lfs diff=lfs merge=lfs -text
97
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep filter=lfs diff=lfs merge=lfs -text
98
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
96
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep filter=lfs diff=lfs merge=lfs -text
97
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep filter=lfs diff=lfs merge=lfs -text
98
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep filter=lfs diff=lfs merge=lfs -text
99
+ granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1.nsys-rep filter=lfs diff=lfs merge=lfs -text
100
+ granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125.nsys-rep filter=lfs diff=lfs merge=lfs -text
101
+ granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep filter=lfs diff=lfs merge=lfs -text
102
+ granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375.nsys-rep filter=lfs diff=lfs merge=lfs -text
granularity_next_b200/tiera_topk_trace/hf_watcher.log CHANGED
@@ -42,3 +42,6 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
42
  uploaded: /workspace/granularity_next_b200/tiera_topk_trace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_topk_trace
43
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
44
  [2026-09-21T05:42:27+00:00] backup pass
 
 
 
 
42
  uploaded: /workspace/granularity_next_b200/tiera_topk_trace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_topk_trace
43
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
44
  [2026-09-21T05:42:27+00:00] backup pass
45
+ uploaded: /workspace/granularity_next_b200/tiera_topk_trace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_topk_trace
46
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
47
+ [2026-09-21T05:52:31+00:00] backup pass
granularity_next_b200/tiera_topk_trace/study.log CHANGED
@@ -186,3 +186,26 @@ driver_repo: /workspace/blackwell_run_scripts @ b8bd100
186
  [05:40:36] TRACE-ONLY G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667 (ranks: 0 1 2 3 4 5 6 7)
187
  [guard] iter 6/10 (70s)
188
  [guard] iter 8/10 (80s)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  [05:40:36] TRACE-ONLY G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667 (ranks: 0 1 2 3 4 5 6 7)
187
  [guard] iter 6/10 (70s)
188
  [guard] iter 8/10 (80s)
189
+ [guard] no progress for 60s at iter 8/10 (past profile window); treating as done
190
+ [guard] nsys stop (flushing report)
191
+ [guard] tearing down process group (outcome=STALLED)
192
+ [05:43:44] trace verified -> G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep (15M, all 8 GPUs + ultraep markers)
193
+ [05:43:44] TRACE-ONLY G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375 (ranks: 0 1 2 3 4 5 6 7)
194
+ [guard] iter 6/10 (70s)
195
+ [guard] iter 8/10 (80s)
196
+ [guard] no progress for 60s at iter 8/10 (past profile window); treating as done
197
+ [guard] nsys stop (flushing report)
198
+ [guard] tearing down process group (outcome=STALLED)
199
+ [05:46:52] trace verified -> G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375.nsys-rep (14M, all 8 GPUs + ultraep markers)
200
+ [05:46:52] TRACE-ONLY G4_K10__ultraep__matched__seq4096__gbs16__cap1p1 (ranks: 0 1 2 3 4 5 6 7)
201
+ [guard] iter 6/10 (70s)
202
+ [guard] iter 8/10 (80s)
203
+ [guard] no progress for 60s at iter 8/10 (past profile window); treating as done
204
+ [guard] nsys stop (flushing report)
205
+ [guard] tearing down process group (outcome=STALLED)
206
+ [05:50:00] trace verified -> G4_K10__ultraep__matched__seq4096__gbs16__cap1p1.nsys-rep (14M, all 8 GPUs + ultraep markers)
207
+ [05:50:00] TRACE-ONLY G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125 (ranks: 0 1 2 3 4 5 6 7)
208
+ [guard] iter 7/10 (70s)
209
+ [guard] iter 8/10 (80s)
210
+ [guard] no progress for 60s at iter 8/10 (past profile window); treating as done
211
+ [guard] nsys stop (flushing report)
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42c78a03428c849cf0720c9222fb05ff3b78bd5449fb232a9b88928a3b237739
3
+ size 14616635
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/run.env ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=G4_K10
3
+ family=tierA
4
+ source_model=
5
+ standard_num_layers=
6
+ base_rung=G4
7
+ repo=/workspace/Megatron-UltraEP
8
+ branch=ultraep-bracing-regions-through-kernels
9
+ commit=03d57ef7
10
+ num_layers=16
11
+ hidden=4096
12
+ ffn_hidden=14336
13
+ moe_ffn_hidden=896
14
+ num_experts=128
15
+ topk=10
16
+ experts_per_rank=16
17
+ expert_params=11010048
18
+ heads=32
19
+ kv_groups=8
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ q_lora_rank=
27
+ kv_lora_rank=512
28
+ qk_head_dim=128
29
+ qk_pos_emb_head_dim=64
30
+ v_head_dim=128
31
+ dense_first_layers=0
32
+ moe_layers=16
33
+ rotary_base=1000000
34
+ rotary_percent=1.0
35
+ rotary_scaling=
36
+ mscale=
37
+ mscale_all_dim=
38
+ qkv_bias=0
39
+ router_balancing=aux_loss
40
+ aux_loss_coeff=1e-2
41
+ router_pre_softmax=0
42
+ router_fusion=0
43
+ router_dtype=fp32
44
+ router_groups=
45
+ router_group_topk=
46
+ router_score_function=softmax
47
+ router_expert_bias=0
48
+ router_bias_update_rate=
49
+ router_topk_scaling_factor=
50
+ shared_expert_gate=0
51
+ apply_layernorm_1p=0
52
+ attention_output_gate=0
53
+ linear_attention_type=
54
+ linear_attention_freq=
55
+ linear_conv_kernel_dim=
56
+ linear_key_head_dim=
57
+ linear_value_head_dim=
58
+ linear_num_key_heads=
59
+ linear_num_value_heads=
60
+ mtp_num_layers=0
61
+ mtp_loss_scaling_factor=0.1
62
+ no_weight_decay_cond_type=
63
+ max_position_embeddings=32768
64
+ init_method_std=0.01
65
+ device_initiated_grouped_gemm=1
66
+ norm_epsilon=1e-5
67
+ seq_length=4096
68
+ micro_batch_size=1
69
+ global_batch_size=16
70
+ num_microbatches=2
71
+ assignments_per_rank=40960
72
+ mean_tokens_per_expert=2560
73
+ slot_policy=matched
74
+ slots_per_rank=16
75
+ capacity_mode=static
76
+ capacity=1.1
77
+ cuda_graph=1
78
+ probe_mode=0
79
+ async_grad_reduce=0
80
+ branch_role=timing
81
+ memory_dump_mode=none
82
+ memory_snapshot_path=
83
+ cuda_graph_warmup_steps=3
84
+ recompute=none
85
+ force_balance=0
86
+ train_iters=10
87
+ ep_size=8
88
+ echo_dispatch_overlap=1
89
+ echo_dispatch_recompute=0
90
+ ultraep_quota_min_tokens=320
91
+ ultraep_weight_sync_join=
92
+ profile_enable=1
93
+ kernel_trace=1
94
+ fp8_mode=mxfp8
95
+ fp8_recipe=mxfp8
96
+ training_seed=1234
97
+ learning_rate=1e-4
98
+ min_learning_rate=1.0e-5
99
+ lr_warmup_iters=5
100
+ state_dump_enable=0
101
+ state_dump_dir=
102
+ moonep_assert_no_pending_reduce=0
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1.nsys-rep
2
+ kernels : 186566
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 26624 ultraep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61a2ba03359a49ef3af3e20339235b2d1dcc9727456ac556ccd12fa7f7b575ef
3
+ size 14847149
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/run.env ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=G4_K16
3
+ family=tierA
4
+ source_model=
5
+ standard_num_layers=
6
+ base_rung=G4
7
+ repo=/workspace/Megatron-UltraEP
8
+ branch=ultraep-bracing-regions-through-kernels
9
+ commit=03d57ef7
10
+ num_layers=16
11
+ hidden=4096
12
+ ffn_hidden=14336
13
+ moe_ffn_hidden=896
14
+ num_experts=128
15
+ topk=16
16
+ experts_per_rank=16
17
+ expert_params=11010048
18
+ heads=32
19
+ kv_groups=8
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ q_lora_rank=
27
+ kv_lora_rank=512
28
+ qk_head_dim=128
29
+ qk_pos_emb_head_dim=64
30
+ v_head_dim=128
31
+ dense_first_layers=0
32
+ moe_layers=16
33
+ rotary_base=1000000
34
+ rotary_percent=1.0
35
+ rotary_scaling=
36
+ mscale=
37
+ mscale_all_dim=
38
+ qkv_bias=0
39
+ router_balancing=aux_loss
40
+ aux_loss_coeff=1e-2
41
+ router_pre_softmax=0
42
+ router_fusion=0
43
+ router_dtype=fp32
44
+ router_groups=
45
+ router_group_topk=
46
+ router_score_function=softmax
47
+ router_expert_bias=0
48
+ router_bias_update_rate=
49
+ router_topk_scaling_factor=
50
+ shared_expert_gate=0
51
+ apply_layernorm_1p=0
52
+ attention_output_gate=0
53
+ linear_attention_type=
54
+ linear_attention_freq=
55
+ linear_conv_kernel_dim=
56
+ linear_key_head_dim=
57
+ linear_value_head_dim=
58
+ linear_num_key_heads=
59
+ linear_num_value_heads=
60
+ mtp_num_layers=0
61
+ mtp_loss_scaling_factor=0.1
62
+ no_weight_decay_cond_type=
63
+ max_position_embeddings=32768
64
+ init_method_std=0.01
65
+ device_initiated_grouped_gemm=1
66
+ norm_epsilon=1e-5
67
+ seq_length=4096
68
+ micro_batch_size=1
69
+ global_batch_size=16
70
+ num_microbatches=2
71
+ assignments_per_rank=65536
72
+ mean_tokens_per_expert=4096
73
+ slot_policy=matched
74
+ slots_per_rank=16
75
+ capacity_mode=static
76
+ capacity=1.08203125
77
+ cuda_graph=1
78
+ probe_mode=0
79
+ async_grad_reduce=0
80
+ branch_role=timing
81
+ memory_dump_mode=none
82
+ memory_snapshot_path=
83
+ cuda_graph_warmup_steps=3
84
+ recompute=none
85
+ force_balance=0
86
+ train_iters=10
87
+ ep_size=8
88
+ echo_dispatch_overlap=1
89
+ echo_dispatch_recompute=0
90
+ ultraep_quota_min_tokens=512
91
+ ultraep_weight_sync_join=
92
+ profile_enable=1
93
+ kernel_trace=1
94
+ fp8_mode=mxfp8
95
+ fp8_recipe=mxfp8
96
+ training_seed=1234
97
+ learning_rate=1e-4
98
+ min_learning_rate=1.0e-5
99
+ lr_warmup_iters=5
100
+ state_dump_enable=0
101
+ state_dump_dir=
102
+ moonep_assert_no_pending_reduce=0
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:705ef359244988c0e0087a3e3adaca5916ecef872af7a38210c362aee169a020
3
+ size 14738224
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/train.log CHANGED
@@ -1520,3 +1520,52 @@ CUDA graph capture done!!!
1520
  [2026-09-21 05:41:47] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 297.5 | throughput per GPU (TFLOP/s/GPU): 363.9 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.018421E+01 | loss scale: 1.0 | grad norm: 22.919 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 26240 | min safe capacity: 1.0677 |
1521
  [2026-09-21 05:41:50] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3234.5 | throughput per GPU (TFLOP/s/GPU): 33.5 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.914703E+00 | loss scale: 1.0 | grad norm: 7.133 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 26240 | min safe capacity: 1.0677 |
1522
  [2026-09-21 05:41:50] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 307.8 | throughput per GPU (TFLOP/s/GPU): 351.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.391114E+00 | loss scale: 1.0 | grad norm: 6.074 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 26240 | min safe capacity: 1.0677 |
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1520
  [2026-09-21 05:41:47] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 297.5 | throughput per GPU (TFLOP/s/GPU): 363.9 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.018421E+01 | loss scale: 1.0 | grad norm: 22.919 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 26240 | min safe capacity: 1.0677 |
1521
  [2026-09-21 05:41:50] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3234.5 | throughput per GPU (TFLOP/s/GPU): 33.5 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.914703E+00 | loss scale: 1.0 | grad norm: 7.133 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 26240 | min safe capacity: 1.0677 |
1522
  [2026-09-21 05:41:50] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 307.8 | throughput per GPU (TFLOP/s/GPU): 351.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.391114E+00 | loss scale: 1.0 | grad norm: 6.074 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 26240 | min safe capacity: 1.0677 |
1523
+ Capture range started in the application.
1524
+ Capture range ended in the application.
1525
+ Generating '/tmp/nsys-report-1939.qdstrm'
1526
+
1527
+
1528
+ Collection stop is not allowed in this state.
1529
+ W0921 05:43:27.857000 84761 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1530
+ W0921 05:43:27.857000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85076 closing signal SIGTERM
1531
+ Generated:
1532
+ /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep
1533
+ W0921 05:43:27.859000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85077 closing signal SIGTERM
1534
+ W0921 05:43:27.862000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85078 closing signal SIGTERM
1535
+ W0921 05:43:27.863000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85079 closing signal SIGTERM
1536
+ W0921 05:43:27.869000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85080 closing signal SIGTERM
1537
+ W0921 05:43:27.869000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85082 closing signal SIGTERM
1538
+ W0921 05:43:27.872000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85083 closing signal SIGTERM
1539
+ W0921 05:43:27.874000 84761 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 85084 closing signal SIGTERM
1540
+ Traceback (most recent call last):
1541
+ File "/usr/local/bin/torchrun", line 8, in <module>
1542
+ sys.exit(main())
1543
+ ^^^^^^
1544
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1545
+ return f(*args, **kwargs)
1546
+ ^^^^^^^^^^^^^^^^^^
1547
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1548
+ run(args)
1549
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1550
+ elastic_launch(
1551
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1552
+ return launch_agent(self._config, self._entrypoint, list(args))
1553
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1554
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1555
+ result = agent.run()
1556
+ ^^^^^^^^^^^
1557
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1558
+ result = f(*args, **kwargs)
1559
+ ^^^^^^^^^^^^^^^^^^
1560
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1561
+ result = self._invoke_run(role)
1562
+ ^^^^^^^^^^^^^^^^^^^^^^
1563
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1564
+ time.sleep(monitor_interval)
1565
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1566
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1567
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 84761 got signal: 15
1568
+ The target application terminated. One or more process it created re-parented.
1569
+ Waiting for termination of re-parented processes.
1570
+ Use the `--wait` option to modify this behavior.
1571
+ There are no active sessions.
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep
2
+ kernels : 186566
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 26624 ultraep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f72eceeabe849759508a181d594a0acd9d130461aca634b47dcf8bf0557fca4
3
+ size 14667030
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/run.env ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=G4_K8
3
+ family=tierA
4
+ source_model=
5
+ standard_num_layers=
6
+ base_rung=G4
7
+ repo=/workspace/Megatron-UltraEP
8
+ branch=ultraep-bracing-regions-through-kernels
9
+ commit=03d57ef7
10
+ num_layers=16
11
+ hidden=4096
12
+ ffn_hidden=14336
13
+ moe_ffn_hidden=896
14
+ num_experts=128
15
+ topk=8
16
+ experts_per_rank=16
17
+ expert_params=11010048
18
+ heads=32
19
+ kv_groups=8
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ q_lora_rank=
27
+ kv_lora_rank=512
28
+ qk_head_dim=128
29
+ qk_pos_emb_head_dim=64
30
+ v_head_dim=128
31
+ dense_first_layers=0
32
+ moe_layers=16
33
+ rotary_base=1000000
34
+ rotary_percent=1.0
35
+ rotary_scaling=
36
+ mscale=
37
+ mscale_all_dim=
38
+ qkv_bias=0
39
+ router_balancing=aux_loss
40
+ aux_loss_coeff=1e-2
41
+ router_pre_softmax=0
42
+ router_fusion=0
43
+ router_dtype=fp32
44
+ router_groups=
45
+ router_group_topk=
46
+ router_score_function=softmax
47
+ router_expert_bias=0
48
+ router_bias_update_rate=
49
+ router_topk_scaling_factor=
50
+ shared_expert_gate=0
51
+ apply_layernorm_1p=0
52
+ attention_output_gate=0
53
+ linear_attention_type=
54
+ linear_attention_freq=
55
+ linear_conv_kernel_dim=
56
+ linear_key_head_dim=
57
+ linear_value_head_dim=
58
+ linear_num_key_heads=
59
+ linear_num_value_heads=
60
+ mtp_num_layers=0
61
+ mtp_loss_scaling_factor=0.1
62
+ no_weight_decay_cond_type=
63
+ max_position_embeddings=32768
64
+ init_method_std=0.01
65
+ device_initiated_grouped_gemm=1
66
+ norm_epsilon=1e-5
67
+ seq_length=4096
68
+ micro_batch_size=1
69
+ global_batch_size=16
70
+ num_microbatches=2
71
+ assignments_per_rank=32768
72
+ mean_tokens_per_expert=2048
73
+ slot_policy=matched
74
+ slots_per_rank=16
75
+ capacity_mode=static
76
+ capacity=1.109375
77
+ cuda_graph=1
78
+ probe_mode=0
79
+ async_grad_reduce=0
80
+ branch_role=timing
81
+ memory_dump_mode=none
82
+ memory_snapshot_path=
83
+ cuda_graph_warmup_steps=3
84
+ recompute=none
85
+ force_balance=0
86
+ train_iters=10
87
+ ep_size=8
88
+ echo_dispatch_overlap=1
89
+ echo_dispatch_recompute=0
90
+ ultraep_quota_min_tokens=256
91
+ ultraep_weight_sync_join=
92
+ profile_enable=1
93
+ kernel_trace=1
94
+ fp8_mode=mxfp8
95
+ fp8_recipe=mxfp8
96
+ training_seed=1234
97
+ learning_rate=1e-4
98
+ min_learning_rate=1.0e-5
99
+ lr_warmup_iters=5
100
+ state_dump_enable=0
101
+ state_dump_dir=
102
+ moonep_assert_no_pending_reduce=0
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375.nsys-rep
2
+ kernels : 186570
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 26624 ultraep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/trace_results.tsv CHANGED
@@ -4,3 +4,6 @@ G3_K2 ultraep 1.1875 OK /workspace/granularity_next_b200/tiera_topk_trace/ultrae
4
  G3_K8 ultraep 1.09375 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep
5
  G4_K2 ultraep 1.265625 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep
6
  G4_K4 ultraep 1.171875 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep
 
 
 
 
4
  G3_K8 ultraep 1.09375 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep
5
  G4_K2 ultraep 1.265625 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep
6
  G4_K4 ultraep 1.171875 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep
7
+ G4_K6 ultraep 1.1354166666666667 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep
8
+ G4_K8 ultraep 1.109375 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375.nsys-rep
9
+ G4_K10 ultraep 1.1 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__ultraep__matched__seq4096__gbs16__cap1p1.nsys-rep