AbhishekGhosh1998 commited on
Commit
92e38b9
·
verified ·
1 Parent(s): 31ce049

granularity results granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16 @ 2026-09-21T08:51:37Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/.completed/echo__tierbmem__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T08:46:34+00:00
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/.completed/moonep__tierbmem__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T08:51:30+00:00
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/.completed/ultraep__tierbmem__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T08:49:02+00:00
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/STATUS.txt CHANGED
@@ -1,9 +1,12 @@
1
  study_root: /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16
2
  gbs: 8
3
- updated: 2026-09-21T08:43:54+00:00
4
- current: echo__tierbmem__gbs8
5
 
6
  completed steps:
 
 
 
7
 
8
  failed steps:
9
  (none)
 
1
  study_root: /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16
2
  gbs: 8
3
+ updated: 2026-09-21T08:51:36+00:00
4
+ current: idle
5
 
6
  completed steps:
7
+ echo__tierbmem__gbs8
8
+ moonep__tierbmem__gbs8
9
+ ultraep__tierbmem__gbs8
10
 
11
  failed steps:
12
  (none)
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/collected_all.tsv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
2
+ echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1 rep1 tierbmem 0 0 OLMOE_1B_7B_L16 echo matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 1.08984375 static none 0 1 iteration /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 OK 6 118.60 118.60 271.6 43043 24330
3
+ echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 OLMOE_1B_7B_L16 echo matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 dynamic dynamic none 0 0 iteration /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 FAIL 5 38025 24384
4
+ moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 OLMOE_1B_7B_L16 moonep matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 none none none 0 1 iteration /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 OK 6 118.90 118.90 270.9 36421 17484
5
+ moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 OLMOE_1B_7B_L16 moonep matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 none none none 0 0 iteration /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 FAIL 5 35751 17484
6
+ ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1 rep1 tierbmem 0 0 OLMOE_1B_7B_L16 ultraep matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 1.08984375 static none 0 1 iteration /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 512 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 OK 6 111.20 111.20 289.7 39097 22168
7
+ ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 OLMOE_1B_7B_L16 ultraep matched 8 64 8 1024 8 6291456 16 4096 8 1 32768 4096 dynamic dynamic none 0 0 iteration /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 512 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 1 0 10000 0 OLMoE-1B-7B-0924 16 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 4096 0.02 FAIL 5 37091 22026
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/hf_watcher.log CHANGED
@@ -1,3 +1,5 @@
1
  Already logged in to Hugging Face as: AbhishekGhosh1998
2
  Watching /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16 -> HF every 600s (kill to stop)
3
  [2026-09-21T08:43:55+00:00] backup pass
 
 
 
1
  Already logged in to Hugging Face as: AbhishekGhosh1998
2
  Watching /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16 -> HF every 600s (kill to stop)
3
  [2026-09-21T08:43:55+00:00] backup pass
4
+ uploaded: /workspace/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16
5
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/memory_summary_all.json ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "runs": [
3
+ {
4
+ "run_dir": "echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1",
5
+ "config": "OLMOE_1B_7B_L16",
6
+ "backend": "echo",
7
+ "num_layers": "16",
8
+ "moe_layers": "16",
9
+ "seq_length": "4096",
10
+ "global_batch_size": "8",
11
+ "cuda_graph": "1",
12
+ "capacity_mode": "static",
13
+ "capacity": "1.08984375",
14
+ "memory_dump_mode": "iteration",
15
+ "branch": "better-memory-dump-support",
16
+ "commit": "b7c52442",
17
+ "status": "OK",
18
+ "max_allocated_gib": 22.940572261810303,
19
+ "max_reserved_gib": 23.759765625,
20
+ "fragmentation_gib": 0.8191933631896973,
21
+ "theoretical_weight_optim_gib": 48.31515625,
22
+ "peak_sampled_gib": 42.0341796875,
23
+ "snapshot_pickle_count": 48,
24
+ "snapshot_json_count": 48,
25
+ "snapshot_expected_each": 48,
26
+ "snapshot_complete": 1,
27
+ "effective_peak_gib": 42.0341796875
28
+ },
29
+ {
30
+ "run_dir": "echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1",
31
+ "config": "OLMOE_1B_7B_L16",
32
+ "backend": "echo",
33
+ "num_layers": "16",
34
+ "moe_layers": "16",
35
+ "seq_length": "4096",
36
+ "global_batch_size": "8",
37
+ "cuda_graph": "0",
38
+ "capacity_mode": "dynamic",
39
+ "capacity": "dynamic",
40
+ "memory_dump_mode": "iteration",
41
+ "branch": "better-memory-dump-support",
42
+ "commit": "b7c52442",
43
+ "status": "OK",
44
+ "max_allocated_gib": 22.818925857543945,
45
+ "max_reserved_gib": 23.8125,
46
+ "fragmentation_gib": 0.9935741424560547,
47
+ "theoretical_weight_optim_gib": 48.31515625,
48
+ "peak_sampled_gib": 37.1337890625,
49
+ "snapshot_pickle_count": 40,
50
+ "snapshot_json_count": 40,
51
+ "snapshot_expected_each": 40,
52
+ "snapshot_complete": 1,
53
+ "effective_peak_gib": 37.1337890625
54
+ },
55
+ {
56
+ "run_dir": "moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1",
57
+ "config": "OLMOE_1B_7B_L16",
58
+ "backend": "moonep",
59
+ "num_layers": "16",
60
+ "moe_layers": "16",
61
+ "seq_length": "4096",
62
+ "global_batch_size": "8",
63
+ "cuda_graph": "1",
64
+ "capacity_mode": "none",
65
+ "capacity": "none",
66
+ "memory_dump_mode": "iteration",
67
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
68
+ "commit": "7949f384",
69
+ "status": "OK",
70
+ "max_allocated_gib": 16.60142183303833,
71
+ "max_reserved_gib": 17.07421875,
72
+ "fragmentation_gib": 0.4727969169616699,
73
+ "theoretical_weight_optim_gib": 48.31515625,
74
+ "peak_sampled_gib": 35.5673828125,
75
+ "snapshot_pickle_count": 48,
76
+ "snapshot_json_count": 48,
77
+ "snapshot_expected_each": 48,
78
+ "snapshot_complete": 1,
79
+ "effective_peak_gib": 35.5673828125
80
+ },
81
+ {
82
+ "run_dir": "moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1",
83
+ "config": "OLMOE_1B_7B_L16",
84
+ "backend": "moonep",
85
+ "num_layers": "16",
86
+ "moe_layers": "16",
87
+ "seq_length": "4096",
88
+ "global_batch_size": "8",
89
+ "cuda_graph": "0",
90
+ "capacity_mode": "none",
91
+ "capacity": "none",
92
+ "memory_dump_mode": "iteration",
93
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
94
+ "commit": "7949f384",
95
+ "status": "OK",
96
+ "max_allocated_gib": 16.60142183303833,
97
+ "max_reserved_gib": 17.07421875,
98
+ "fragmentation_gib": 0.4727969169616699,
99
+ "theoretical_weight_optim_gib": 48.31515625,
100
+ "peak_sampled_gib": 34.9130859375,
101
+ "snapshot_pickle_count": 40,
102
+ "snapshot_json_count": 40,
103
+ "snapshot_expected_each": 40,
104
+ "snapshot_complete": 1,
105
+ "effective_peak_gib": 34.9130859375
106
+ },
107
+ {
108
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1",
109
+ "config": "OLMOE_1B_7B_L16",
110
+ "backend": "ultraep",
111
+ "num_layers": "16",
112
+ "moe_layers": "16",
113
+ "seq_length": "4096",
114
+ "global_batch_size": "8",
115
+ "cuda_graph": "1",
116
+ "capacity_mode": "static",
117
+ "capacity": "1.08984375",
118
+ "memory_dump_mode": "iteration",
119
+ "branch": "ultraep-better-memory-dump-support",
120
+ "commit": "c3a8a8a7",
121
+ "status": "OK",
122
+ "max_allocated_gib": 21.23912763595581,
123
+ "max_reserved_gib": 21.6484375,
124
+ "fragmentation_gib": 0.40930986404418945,
125
+ "theoretical_weight_optim_gib": 48.31515625,
126
+ "peak_sampled_gib": 38.1806640625,
127
+ "snapshot_pickle_count": 48,
128
+ "snapshot_json_count": 48,
129
+ "snapshot_expected_each": 48,
130
+ "snapshot_complete": 1,
131
+ "effective_peak_gib": 38.1806640625
132
+ },
133
+ {
134
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1",
135
+ "config": "OLMOE_1B_7B_L16",
136
+ "backend": "ultraep",
137
+ "num_layers": "16",
138
+ "moe_layers": "16",
139
+ "seq_length": "4096",
140
+ "global_batch_size": "8",
141
+ "cuda_graph": "0",
142
+ "capacity_mode": "dynamic",
143
+ "capacity": "dynamic",
144
+ "memory_dump_mode": "iteration",
145
+ "branch": "ultraep-better-memory-dump-support",
146
+ "commit": "c3a8a8a7",
147
+ "status": "OK",
148
+ "max_allocated_gib": 21.081180095672607,
149
+ "max_reserved_gib": 21.509765625,
150
+ "fragmentation_gib": 0.4285855293273926,
151
+ "theoretical_weight_optim_gib": 48.31515625,
152
+ "peak_sampled_gib": 36.2216796875,
153
+ "snapshot_pickle_count": 40,
154
+ "snapshot_json_count": 40,
155
+ "snapshot_expected_each": 40,
156
+ "snapshot_complete": 1,
157
+ "effective_peak_gib": 36.2216796875
158
+ }
159
+ ],
160
+ "comparisons": [
161
+ {
162
+ "config": "OLMOE_1B_7B_L16",
163
+ "backend": "echo",
164
+ "dynamic_nocg_status": "OK",
165
+ "dynamic_nocg_peak_gib": 37.1337890625,
166
+ "cap_1p08984375_status": "OK",
167
+ "cap_1p08984375_peak_gib": 42.0341796875
168
+ },
169
+ {
170
+ "config": "OLMOE_1B_7B_L16",
171
+ "backend": "moonep",
172
+ "moon_graph_on_status": "OK",
173
+ "moon_graph_on_peak_gib": 35.5673828125,
174
+ "moon_graph_off_status": "OK",
175
+ "moon_graph_off_peak_gib": 34.9130859375,
176
+ "moon_graph_delta_gib": 0.654296875
177
+ },
178
+ {
179
+ "config": "OLMOE_1B_7B_L16",
180
+ "backend": "ultraep",
181
+ "dynamic_nocg_status": "OK",
182
+ "dynamic_nocg_peak_gib": 36.2216796875,
183
+ "cap_1p08984375_status": "OK",
184
+ "cap_1p08984375_peak_gib": 38.1806640625
185
+ }
186
+ ]
187
+ }
granularity_next_b200/tierb_models/memory/OLMOE_1B_7B_L16/memory_summary_all.tsv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ run_dir config backend cuda_graph capacity_mode capacity status effective_peak_gib max_allocated_gib max_reserved_gib fragmentation_gib peak_sampled_gib snapshot_pickle_count snapshot_json_count snapshot_expected_each snapshot_complete branch commit
2
+ echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1 OLMOE_1B_7B_L16 echo 1 static 1.08984375 OK 42.0341796875 22.940572261810303 23.759765625 0.8191933631896973 42.0341796875 48 48 48 1 better-memory-dump-support b7c52442
3
+ echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 OLMOE_1B_7B_L16 echo 0 dynamic dynamic OK 37.1337890625 22.818925857543945 23.8125 0.9935741424560547 37.1337890625 40 40 40 1 better-memory-dump-support b7c52442
4
+ moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1 OLMOE_1B_7B_L16 moonep 1 none none OK 35.5673828125 16.60142183303833 17.07421875 0.4727969169616699 35.5673828125 48 48 48 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
5
+ moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1 OLMOE_1B_7B_L16 moonep 0 none none OK 34.9130859375 16.60142183303833 17.07421875 0.4727969169616699 34.9130859375 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
6
+ ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1 OLMOE_1B_7B_L16 ultraep 1 static 1.08984375 OK 38.1806640625 21.23912763595581 21.6484375 0.40930986404418945 38.1806640625 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
7
+ ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 OLMOE_1B_7B_L16 ultraep 0 dynamic dynamic OK 36.2216796875 21.081180095672607 21.509765625 0.4285855293273926 36.2216796875 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7