granularity results granularity_next_b200/tierb_models/traces/KIMI_K2_L2 @ 2026-09-21T08:23:01Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
Browse files- granularity_next_b200/tierb_models/traces/KIMI_K2_L2/.completed/moonep__tierbtrace__gbs8 +1 -0
- granularity_next_b200/tierb_models/traces/KIMI_K2_L2/.completed/ultraep__tierbtrace__gbs8 +1 -0
- granularity_next_b200/tierb_models/traces/KIMI_K2_L2/STATUS.txt +4 -2
- granularity_next_b200/tierb_models/traces/KIMI_K2_L2/collected_all.tsv +4 -0
- granularity_next_b200/tierb_models/traces/KIMI_K2_L2/hf_watcher.log +2 -0
- granularity_next_b200/tierb_models/traces/KIMI_K2_L2/steps.tsv +2 -0
- granularity_next_b200/tierb_models/traces/KIMI_K2_L2/study.log +65 -0
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/.completed/moonep__tierbtrace__gbs8
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-09-21T08:22:57+00:00
|
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/.completed/ultraep__tierbtrace__gbs8
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-09-21T08:16:21+00:00
|
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/STATUS.txt
CHANGED
|
@@ -1,10 +1,12 @@
|
|
| 1 |
study_root: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
|
| 2 |
gbs: 8
|
| 3 |
-
updated: 2026-09-21T08:
|
| 4 |
-
current:
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__tierbtrace__gbs8
|
|
|
|
|
|
|
| 8 |
|
| 9 |
failed steps:
|
| 10 |
(none)
|
|
|
|
| 1 |
study_root: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
|
| 2 |
gbs: 8
|
| 3 |
+
updated: 2026-09-21T08:23:00+00:00
|
| 4 |
+
current: idle
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__tierbtrace__gbs8
|
| 8 |
+
moonep__tierbtrace__gbs8
|
| 9 |
+
ultraep__tierbtrace__gbs8
|
| 10 |
|
| 11 |
failed steps:
|
| 12 |
(none)
|
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/collected_all.tsv
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
|
| 2 |
+
echo/tierbtrace/tierbtrace_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/nsys tierbtrace 1 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 10 8456.02 118.90 16983.30 175.5 74666
|
| 3 |
+
moonep/tierbtrace/tierbtrace_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/nsys tierbtrace 1 1 KIMI_K2_L2 moonep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 10 8608.34 108.60 17476.74 192.3 45790
|
| 4 |
+
ultraep/tierbtrace/tierbtrace_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/nsys tierbtrace 1 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 85 0 ultraep-bracing-regions-through-kernels 03d57ef7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 10 8447.80 111.90 17012.71 188.0 54394
|
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/hf_watcher.log
CHANGED
|
@@ -4,3 +4,5 @@ Watching /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> HF e
|
|
| 4 |
uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
|
| 5 |
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 6 |
[2026-09-21T08:13:03+00:00] backup pass
|
|
|
|
|
|
|
|
|
| 4 |
uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
|
| 5 |
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 6 |
[2026-09-21T08:13:03+00:00] backup pass
|
| 7 |
+
uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
|
| 8 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/steps.tsv
CHANGED
|
@@ -1,2 +1,4 @@
|
|
| 1 |
status campaign phase gbs started ended result_dir
|
| 2 |
OK echo tierbtrace 8 2026-09-21T08:03:00+00:00 2026-09-21T08:09:43+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/echo/tierbtrace
|
|
|
|
|
|
|
|
|
| 1 |
status campaign phase gbs started ended result_dir
|
| 2 |
OK echo tierbtrace 8 2026-09-21T08:03:00+00:00 2026-09-21T08:09:43+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/echo/tierbtrace
|
| 3 |
+
OK ultraep tierbtrace 8 2026-09-21T08:09:47+00:00 2026-09-21T08:16:21+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
|
| 4 |
+
OK moonep tierbtrace 8 2026-09-21T08:16:25+00:00 2026-09-21T08:22:57+00:00 /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
|
granularity_next_b200/tierb_models/traces/KIMI_K2_L2/study.log
CHANGED
|
@@ -48,3 +48,68 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 48 |
results -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
|
| 49 |
==============================================================
|
| 50 |
[08:10:02] building ultraep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
results -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
|
| 49 |
==============================================================
|
| 50 |
[08:10:02] building ultraep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
|
| 51 |
+
[08:13:59] marker extension ready: _ultraep_trace_C.cpython-312-x86_64-linux-gnu.so
|
| 52 |
+
[08:13:59] TRACE-ONLY KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125 (ranks: 0 1 2 3 4 5 6 7)
|
| 53 |
+
[guard] iter 8/10 (70s)
|
| 54 |
+
[guard] iter 10/10 (110s)
|
| 55 |
+
[guard] nsys stop (flushing report)
|
| 56 |
+
[08:16:21] trace verified -> KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125.nsys-rep (3.3M, all 8 GPUs + ultraep markers)
|
| 57 |
+
--------------------------------------------------------------------------
|
| 58 |
+
|
| 59 |
+
==============================================================
|
| 60 |
+
SWEEP COMPLETE ok=1 failed=0
|
| 61 |
+
==============================================================
|
| 62 |
+
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 63 |
+
|
| 64 |
+
Results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8/results.tsv
|
| 65 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace/tierbtrace_gbs8
|
| 66 |
+
==============================================================
|
| 67 |
+
|
| 68 |
+
Campaign results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
|
| 69 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace --compare
|
| 70 |
+
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 71 |
+
uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/ultraep/tierbtrace
|
| 72 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 73 |
+
|
| 74 |
+
=== STEP moonep__tierbtrace__gbs8
|
| 75 |
+
==============================================================
|
| 76 |
+
CAMPAIGN: moonep backend=moonep phase=tierbtrace
|
| 77 |
+
Megatron: /workspace/Megatron-MoE-Echo @ moonep_backend_async_grad_reduce
|
| 78 |
+
GBS=8 seq=4096 repeats=3 nsys=1
|
| 79 |
+
results -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
|
| 80 |
+
==============================================================
|
| 81 |
+
[08:16:39] building moonep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
|
| 82 |
+
[08:20:35] marker extension ready: _moonep_trace_C.cpython-312-x86_64-linux-gnu.so
|
| 83 |
+
[08:20:35] TRACE-ONLY KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async (ranks: 0 1 2 3 4 5 6 7)
|
| 84 |
+
[guard] iter 3/10 (60s)
|
| 85 |
+
[guard] iter 8/10 (70s)
|
| 86 |
+
[guard] iter 10/10 (110s)
|
| 87 |
+
[guard] nsys stop (flushing report)
|
| 88 |
+
[08:22:57] trace verified -> KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep (3.2M, all 8 GPUs + moonep markers)
|
| 89 |
+
--------------------------------------------------------------------------
|
| 90 |
+
|
| 91 |
+
==============================================================
|
| 92 |
+
SWEEP COMPLETE ok=1 failed=0
|
| 93 |
+
==============================================================
|
| 94 |
+
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 95 |
+
|
| 96 |
+
Results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace/tierbtrace_gbs8/results.tsv
|
| 97 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace/tierbtrace_gbs8
|
| 98 |
+
==============================================================
|
| 99 |
+
|
| 100 |
+
Campaign results: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
|
| 101 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace --compare
|
| 102 |
+
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 103 |
+
uploaded: /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/moonep/tierbtrace
|
| 104 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 105 |
+
|
| 106 |
+
=== ROLL-UP
|
| 107 |
+
no successful runs to compare
|
| 108 |
+
3 runs ok=3 oom=0 -> /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2/collected_all.tsv
|
| 109 |
+
|
| 110 |
+
(excluded 3 nsys-traced run(s) from the timing comparison)
|
| 111 |
+
|
| 112 |
+
no probe telemetry found under /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
|
| 113 |
+
expected instrumented branches: token-drop-analysis / ultraep-token-drop-analysis / moonep_backend_token_routing_analysis
|
| 114 |
+
no memory-dump runs found under /workspace/granularity_next_b200/tierb_models/traces/KIMI_K2_L2
|
| 115 |
+
Already logged in to Hugging Face as: AbhishekGhosh1998
|