AbhishekGhosh1998 commited on
Commit
1dbe36a
·
verified ·
1 Parent(s): 6e9baaa

granularity results granularity_next_b200/tierb_models/traces/KIMI_K2_L2 @ 2026-09-21T08:23:01Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/.completed/moonep__tierbtrace__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T08:22:57+00:00
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/.completed/ultraep__tierbtrace__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T08:16:21+00:00
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/STATUS.txt CHANGED
@@ -1,10 +1,12 @@
1
  study_root: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
2
  gbs: 8
3
- updated: 2026-09-21T08:09:47+00:00
4
- current: ultraep__tierbtrace__gbs8
5
 
6
  completed steps:
7
  echo__tierbtrace__gbs8
 
 
8
 
9
  failed steps:
10
  (none)
 
1
  study_root: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
2
  gbs: 8
3
+ updated: 2026-09-21T08:23:00+00:00
4
+ current: idle
5
 
6
  completed steps:
7
  echo__tierbtrace__gbs8
8
+ moonep__tierbtrace__gbs8
9
+ ultraep__tierbtrace__gbs8
10
 
11
  failed steps:
12
  (none)
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/collected_all.tsv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
2
+ echo/tierbtrace/tierbtrace_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/nsys tierbtrace 1 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 10 8456.02 118.90 16983.30 175.5 74666
3
+ moonep/tierbtrace/tierbtrace_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/nsys tierbtrace 1 1 KIMI_K2_L2 moonep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 10 8608.34 108.60 17476.74 192.3 45790
4
+ ultraep/tierbtrace/tierbtrace_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/nsys tierbtrace 1 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 85 0 ultraep-bracing-regions-through-kernels 03d57ef7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 10 8447.80 111.90 17012.71 188.0 54394
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/hf_watcher.log CHANGED
@@ -4,3 +4,5 @@ Watching /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> HF e
4
  uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
5
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
6
  [2026-09-21T08:13:03+00:00] backup pass
 
 
 
4
  uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
5
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
6
  [2026-09-21T08:13:03+00:00] backup pass
7
+ uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
8
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/steps.tsv CHANGED
@@ -1,2 +1,4 @@
1
  status campaign phase gbs started ended result_dir
2
  OK echo tierbtrace 8 2026-09-21T08:03:00+00:00 2026-09-21T08:09:43+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/echo/tierbtrace
 
 
 
1
  status campaign phase gbs started ended result_dir
2
  OK echo tierbtrace 8 2026-09-21T08:03:00+00:00 2026-09-21T08:09:43+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/echo/tierbtrace
3
+ OK ultraep tierbtrace 8 2026-09-21T08:09:47+00:00 2026-09-21T08:16:21+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
4
+ OK moonep tierbtrace 8 2026-09-21T08:16:25+00:00 2026-09-21T08:22:57+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/study.log CHANGED
@@ -48,3 +48,68 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
48
  results -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
49
  ==============================================================
50
  [08:10:02] building ultraep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  results -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
49
  ==============================================================
50
  [08:10:02] building ultraep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
51
+ [08:13:59] marker extension ready: _ultraep_trace_C.cpython-312-x86_64-linux-gnu.so
52
+ [08:13:59] TRACE-ONLY KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125 (ranks: 0 1 2 3 4 5 6 7)
53
+ [guard] iter 8/10 (70s)
54
+ [guard] iter 10/10 (110s)
55
+ [guard] nsys stop (flushing report)
56
+ [08:16:21] trace verified -> KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125.nsys-rep (3.3M, all 8 GPUs + ultraep markers)
57
+ --------------------------------------------------------------------------
58
+
59
+ ==============================================================
60
+ SWEEP COMPLETE ok=1 failed=0
61
+ ==============================================================
62
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
63
+
64
+ Results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8/results.tsv
65
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8
66
+ ==============================================================
67
+
68
+ Campaign results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
69
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace --compare
70
+ Already logged in to Hugging Face as: AbhishekGhosh1998
71
+ uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
72
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
73
+
74
+ === STEP moonep__tierbtrace__gbs8
75
+ ==============================================================
76
+ CAMPAIGN: moonep backend=moonep phase=tierbtrace
77
+ Megatron: /workspace/Megatron-MoE-Echo @ moonep_backend_async_grad_reduce
78
+ GBS=8 seq=4096 repeats=3 nsys=1
79
+ results -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
80
+ ==============================================================
81
+ [08:16:39] building moonep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
82
+ [08:20:35] marker extension ready: _moonep_trace_C.cpython-312-x86_64-linux-gnu.so
83
+ [08:20:35] TRACE-ONLY KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async (ranks: 0 1 2 3 4 5 6 7)
84
+ [guard] iter 3/10 (60s)
85
+ [guard] iter 8/10 (70s)
86
+ [guard] iter 10/10 (110s)
87
+ [guard] nsys stop (flushing report)
88
+ [08:22:57] trace verified -> KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep (3.2M, all 8 GPUs + moonep markers)
89
+ --------------------------------------------------------------------------
90
+
91
+ ==============================================================
92
+ SWEEP COMPLETE ok=1 failed=0
93
+ ==============================================================
94
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
95
+
96
+ Results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace/tierbtrace_gbs8/results.tsv
97
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace/tierbtrace_gbs8
98
+ ==============================================================
99
+
100
+ Campaign results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
101
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace --compare
102
+ Already logged in to Hugging Face as: AbhishekGhosh1998
103
+ uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
104
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
105
+
106
+ === ROLL-UP
107
+ no successful runs to compare
108
+ 3 runs ok=3 oom=0 -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/collected_all.tsv
109
+
110
+ (excluded 3 nsys-traced run(s) from the timing comparison)
111
+
112
+ no probe telemetry found under /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
113
+ expected instrumented branches: token-drop-analysis / ultraep-token-drop-analysis / moonep_backend_token_routing_analysis
114
+ no memory-dump runs found under /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
115
+ Already logged in to Hugging Face as: AbhishekGhosh1998