AbhishekGhosh1998 commited on
Commit
0bedcdc
·
verified ·
1 Parent(s): 3e531b2

granularity results granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16 @ 2026-09-21T09:11:38Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/.completed/moonep__tierbtrace__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T09:11:33+00:00
granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/.completed/ultraep__tierbtrace__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T09:05:08+00:00
granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/STATUS.txt CHANGED
@@ -1,10 +1,12 @@
1
  study_root: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16
2
  gbs: 8
3
- updated: 2026-09-21T08:58:29+00:00
4
- current: ultraep__tierbtrace__gbs8
5
 
6
  completed steps:
7
  echo__tierbtrace__gbs8
 
 
8
 
9
  failed steps:
10
  (none)
 
1
  study_root: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16
2
  gbs: 8
3
+ updated: 2026-09-21T09:11:37+00:00
4
+ current: idle
5
 
6
  completed steps:
7
  echo__tierbtrace__gbs8
8
+ moonep__tierbtrace__gbs8
9
+ ultraep__tierbtrace__gbs8
10
 
11
  failed steps:
12
  (none)
granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/collected_all.tsv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
2
+ echo/tierbtrace/tierbtrace_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/nsys tierbtrace 1 0 OLMOE_1B_7B_L16 echo matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 1.08984375 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 OK 10 8847.32 113.80 17926.88 157.6 24202
3
+ moonep/tierbtrace/tierbtrace_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/nsys tierbtrace 1 1 OLMOE_1B_7B_L16 moonep matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 OK 10 8801.34 107.00 17917.37 170.0 17484
4
+ ultraep/tierbtrace/tierbtrace_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys tierbtrace 1 0 OLMOE_1B_7B_L16 ultraep matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 1.08984375 static none 0 1 none 512 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 OK 10 8582.14 105.80 17447.92 174.9 22168
granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/hf_watcher.log CHANGED
@@ -4,3 +4,5 @@ Watching /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16 ->
4
  uploaded: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16
5
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
6
  [2026-09-21T09:01:45+00:00] backup pass
 
 
 
4
  uploaded: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16
5
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
6
  [2026-09-21T09:01:45+00:00] backup pass
7
+ uploaded: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16
8
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/steps.tsv CHANGED
@@ -1,2 +1,4 @@
1
  status campaign phase gbs started ended result_dir
2
  OK echo tierbtrace 8 2026-09-21T08:51:43+00:00 2026-09-21T08:58:25+00:00 /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/echo/tierbtrace
 
 
 
1
  status campaign phase gbs started ended result_dir
2
  OK echo tierbtrace 8 2026-09-21T08:51:43+00:00 2026-09-21T08:58:25+00:00 /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/echo/tierbtrace
3
+ OK ultraep tierbtrace 8 2026-09-21T08:58:29+00:00 2026-09-21T09:05:08+00:00 /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace
4
+ OK moonep tierbtrace 8 2026-09-21T09:05:12+00:00 2026-09-21T09:11:33+00:00 /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace
granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/study.log CHANGED
@@ -49,3 +49,68 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
49
  results -> /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace
50
  ==============================================================
51
  [08:58:43] building ultraep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  results -> /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace
50
  ==============================================================
51
  [08:58:43] building ultraep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
52
+ [09:02:45] marker extension ready: _ultraep_trace_C.cpython-312-x86_64-linux-gnu.so
53
+ [09:02:45] TRACE-ONLY OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375 (ranks: 0 1 2 3 4 5 6 7)
54
+ [guard] iter 3/10 (60s)
55
+ [guard] iter 8/10 (70s)
56
+ [guard] iter 10/10 (110s)
57
+ [guard] nsys stop (flushing report)
58
+ [09:05:08] trace verified -> OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375.nsys-rep (7.3M, all 8 GPUs + ultraep markers)
59
+ --------------------------------------------------------------------------
60
+
61
+ ==============================================================
62
+ SWEEP COMPLETE ok=1 failed=0
63
+ ==============================================================
64
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
65
+
66
+ Results: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace/tierbtrace_gbs8/results.tsv
67
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace/tierbtrace_gbs8
68
+ ==============================================================
69
+
70
+ Campaign results: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace
71
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace --compare
72
+ Already logged in to Hugging Face as: AbhishekGhosh1998
73
+ uploaded: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/ultraep/tierbtrace
74
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
75
+
76
+ === STEP moonep__tierbtrace__gbs8
77
+ ==============================================================
78
+ CAMPAIGN: moonep backend=moonep phase=tierbtrace
79
+ Megatron: /workspace/Megatron-MoE-Echo @ moonep_backend_async_grad_reduce
80
+ GBS=8 seq=4096 repeats=3 nsys=1
81
+ results -> /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace
82
+ ==============================================================
83
+ [09:05:26] building moonep marker extension (log: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace/tierbtrace_gbs8/trace_extension_build.log)
84
+ [09:09:21] marker extension ready: _moonep_trace_C.cpython-312-x86_64-linux-gnu.so
85
+ [09:09:21] TRACE-ONLY OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async (ranks: 0 1 2 3 4 5 6 7)
86
+ [guard] iter 8/10 (60s)
87
+ [guard] iter 10/10 (100s)
88
+ [guard] nsys stop (flushing report)
89
+ [09:11:33] trace verified -> OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep (7.0M, all 8 GPUs + moonep markers)
90
+ --------------------------------------------------------------------------
91
+
92
+ ==============================================================
93
+ SWEEP COMPLETE ok=1 failed=0
94
+ ==============================================================
95
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
96
+
97
+ Results: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace/tierbtrace_gbs8/results.tsv
98
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace/tierbtrace_gbs8
99
+ ==============================================================
100
+
101
+ Campaign results: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace
102
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace --compare
103
+ Already logged in to Hugging Face as: AbhishekGhosh1998
104
+ uploaded: /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/moonep/tierbtrace
105
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
106
+
107
+ === ROLL-UP
108
+ no successful runs to compare
109
+ 3 runs ok=3 oom=0 -> /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16/collected_all.tsv
110
+
111
+ (excluded 3 nsys-traced run(s) from the timing comparison)
112
+
113
+ no probe telemetry found under /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16
114
+ expected instrumented branches: token-drop-analysis / ultraep-token-drop-analysis / moonep_backend_token_routing_analysis
115
+ no memory-dump runs found under /workspace/granularity_next_b200/tierb_models/traces/OLMOE_1B_7B_L16
116
+ Already logged in to Hugging Face as: AbhishekGhosh1998