AbhishekGhosh1998 commited on
Commit
ff3d68a
·
verified ·
1 Parent(s): d36cc23

granularity results granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4 @ 2026-09-21T07:12:34Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/.completed/echo__tierbmem__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T07:07:00+00:00
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/.completed/moonep__tierbmem__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T07:12:25+00:00
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/.completed/ultraep__tierbmem__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T07:09:37+00:00
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/STATUS.txt CHANGED
@@ -1,9 +1,12 @@
1
  study_root: /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4
2
  gbs: 8
3
- updated: 2026-09-21T07:04:10+00:00
4
- current: echo__tierbmem__gbs8
5
 
6
  completed steps:
 
 
 
7
 
8
  failed steps:
9
  (none)
 
1
  study_root: /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4
2
  gbs: 8
3
+ updated: 2026-09-21T07:12:33+00:00
4
+ current: idle
5
 
6
  completed steps:
7
+ echo__tierbmem__gbs8
8
+ moonep__tierbmem__gbs8
9
+ ultraep__tierbmem__gbs8
10
 
11
  failed steps:
12
  (none)
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/collected_all.tsv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
2
+ echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1 rep1 tierbmem 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 iteration /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 6 203.90 203.90 365.2 131075 84018
3
+ echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 dynamic dynamic none 0 0 iteration /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 INCOMPLETE 5 119353 81706
4
+ moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 1 iteration /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 6 198.70 198.70 374.8 97849 60652
5
+ moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 0 iteration /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 FAIL 5 83829 60636
6
+ ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1 rep1 tierbmem 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 iteration /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 128 0 ultraep-better-memory-dump-support c3a8a8a7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 6 199.60 199.60 373.1 101995 68070
7
+ ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 dynamic dynamic none 0 0 iteration /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 128 0 ultraep-better-memory-dump-support c3a8a8a7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 INCOMPLETE 5 102673 68054
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/hf_watcher.log CHANGED
@@ -1,3 +1,5 @@
1
  Already logged in to Hugging Face as: AbhishekGhosh1998
2
  Watching /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4 -> HF every 600s (kill to stop)
3
  [2026-09-21T07:04:11+00:00] backup pass
 
 
 
1
  Already logged in to Hugging Face as: AbhishekGhosh1998
2
  Watching /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4 -> HF every 600s (kill to stop)
3
  [2026-09-21T07:04:11+00:00] backup pass
4
+ uploaded: /workspace/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4
5
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/memory_summary_all.json ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "runs": [
3
+ {
4
+ "run_dir": "echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1",
5
+ "config": "DEEPSEEK_V3_L4",
6
+ "backend": "echo",
7
+ "num_layers": "4",
8
+ "moe_layers": "1",
9
+ "seq_length": "4096",
10
+ "global_batch_size": "8",
11
+ "cuda_graph": "1",
12
+ "capacity_mode": "static",
13
+ "capacity": "1.13671875",
14
+ "memory_dump_mode": "iteration",
15
+ "branch": "better-memory-dump-support",
16
+ "commit": "b7c52442",
17
+ "status": "OK",
18
+ "max_allocated_gib": 68.58609199523926,
19
+ "max_reserved_gib": 82.048828125,
20
+ "fragmentation_gib": 13.462736129760742,
21
+ "theoretical_weight_optim_gib": 97.629130859375,
22
+ "peak_sampled_gib": 128.0029296875,
23
+ "snapshot_pickle_count": 48,
24
+ "snapshot_json_count": 48,
25
+ "snapshot_expected_each": 48,
26
+ "snapshot_complete": 1,
27
+ "effective_peak_gib": 128.0029296875
28
+ },
29
+ {
30
+ "run_dir": "echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1",
31
+ "config": "DEEPSEEK_V3_L4",
32
+ "backend": "echo",
33
+ "num_layers": "4",
34
+ "moe_layers": "1",
35
+ "seq_length": "4096",
36
+ "global_batch_size": "8",
37
+ "cuda_graph": "0",
38
+ "capacity_mode": "dynamic",
39
+ "capacity": "dynamic",
40
+ "memory_dump_mode": "iteration",
41
+ "branch": "better-memory-dump-support",
42
+ "commit": "b7c52442",
43
+ "status": "OK",
44
+ "max_allocated_gib": 69.96967601776123,
45
+ "max_reserved_gib": 79.791015625,
46
+ "fragmentation_gib": 9.82133960723877,
47
+ "theoretical_weight_optim_gib": 97.629130859375,
48
+ "peak_sampled_gib": 116.5556640625,
49
+ "snapshot_pickle_count": 40,
50
+ "snapshot_json_count": 40,
51
+ "snapshot_expected_each": 40,
52
+ "snapshot_complete": 1,
53
+ "effective_peak_gib": 116.5556640625
54
+ },
55
+ {
56
+ "run_dir": "moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1",
57
+ "config": "DEEPSEEK_V3_L4",
58
+ "backend": "moonep",
59
+ "num_layers": "4",
60
+ "moe_layers": "1",
61
+ "seq_length": "4096",
62
+ "global_batch_size": "8",
63
+ "cuda_graph": "1",
64
+ "capacity_mode": "none",
65
+ "capacity": "none",
66
+ "memory_dump_mode": "iteration",
67
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
68
+ "commit": "7949f384",
69
+ "status": "OK",
70
+ "max_allocated_gib": 56.274383544921875,
71
+ "max_reserved_gib": 59.23046875,
72
+ "fragmentation_gib": 2.956085205078125,
73
+ "theoretical_weight_optim_gib": 97.629130859375,
74
+ "peak_sampled_gib": 95.5556640625,
75
+ "snapshot_pickle_count": 48,
76
+ "snapshot_json_count": 48,
77
+ "snapshot_expected_each": 48,
78
+ "snapshot_complete": 1,
79
+ "effective_peak_gib": 95.5556640625
80
+ },
81
+ {
82
+ "run_dir": "moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1",
83
+ "config": "DEEPSEEK_V3_L4",
84
+ "backend": "moonep",
85
+ "num_layers": "4",
86
+ "moe_layers": "1",
87
+ "seq_length": "4096",
88
+ "global_batch_size": "8",
89
+ "cuda_graph": "0",
90
+ "capacity_mode": "none",
91
+ "capacity": "none",
92
+ "memory_dump_mode": "iteration",
93
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
94
+ "commit": "7949f384",
95
+ "status": "OK",
96
+ "max_allocated_gib": 56.25963544845581,
97
+ "max_reserved_gib": 59.21484375,
98
+ "fragmentation_gib": 2.9552083015441895,
99
+ "theoretical_weight_optim_gib": 97.629130859375,
100
+ "peak_sampled_gib": 81.8642578125,
101
+ "snapshot_pickle_count": 40,
102
+ "snapshot_json_count": 40,
103
+ "snapshot_expected_each": 40,
104
+ "snapshot_complete": 1,
105
+ "effective_peak_gib": 81.8642578125
106
+ },
107
+ {
108
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1",
109
+ "config": "DEEPSEEK_V3_L4",
110
+ "backend": "ultraep",
111
+ "num_layers": "4",
112
+ "moe_layers": "1",
113
+ "seq_length": "4096",
114
+ "global_batch_size": "8",
115
+ "cuda_graph": "1",
116
+ "capacity_mode": "static",
117
+ "capacity": "1.13671875",
118
+ "memory_dump_mode": "iteration",
119
+ "branch": "ultraep-better-memory-dump-support",
120
+ "commit": "c3a8a8a7",
121
+ "status": "OK",
122
+ "max_allocated_gib": 64.16005086898804,
123
+ "max_reserved_gib": 66.474609375,
124
+ "fragmentation_gib": 2.314558506011963,
125
+ "theoretical_weight_optim_gib": 97.629130859375,
126
+ "peak_sampled_gib": 99.6044921875,
127
+ "snapshot_pickle_count": 48,
128
+ "snapshot_json_count": 48,
129
+ "snapshot_expected_each": 48,
130
+ "snapshot_complete": 1,
131
+ "effective_peak_gib": 99.6044921875
132
+ },
133
+ {
134
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1",
135
+ "config": "DEEPSEEK_V3_L4",
136
+ "backend": "ultraep",
137
+ "num_layers": "4",
138
+ "moe_layers": "1",
139
+ "seq_length": "4096",
140
+ "global_batch_size": "8",
141
+ "cuda_graph": "0",
142
+ "capacity_mode": "dynamic",
143
+ "capacity": "dynamic",
144
+ "memory_dump_mode": "iteration",
145
+ "branch": "ultraep-better-memory-dump-support",
146
+ "commit": "c3a8a8a7",
147
+ "status": "OK",
148
+ "max_allocated_gib": 64.25602293014526,
149
+ "max_reserved_gib": 66.458984375,
150
+ "fragmentation_gib": 2.2029614448547363,
151
+ "theoretical_weight_optim_gib": 97.629130859375,
152
+ "peak_sampled_gib": 100.2666015625,
153
+ "snapshot_pickle_count": 40,
154
+ "snapshot_json_count": 40,
155
+ "snapshot_expected_each": 40,
156
+ "snapshot_complete": 1,
157
+ "effective_peak_gib": 100.2666015625
158
+ }
159
+ ],
160
+ "comparisons": [
161
+ {
162
+ "config": "DEEPSEEK_V3_L4",
163
+ "backend": "echo",
164
+ "dynamic_nocg_status": "OK",
165
+ "dynamic_nocg_peak_gib": 116.5556640625,
166
+ "cap_1p13671875_status": "OK",
167
+ "cap_1p13671875_peak_gib": 128.0029296875
168
+ },
169
+ {
170
+ "config": "DEEPSEEK_V3_L4",
171
+ "backend": "moonep",
172
+ "moon_graph_on_status": "OK",
173
+ "moon_graph_on_peak_gib": 95.5556640625,
174
+ "moon_graph_off_status": "OK",
175
+ "moon_graph_off_peak_gib": 81.8642578125,
176
+ "moon_graph_delta_gib": 13.69140625
177
+ },
178
+ {
179
+ "config": "DEEPSEEK_V3_L4",
180
+ "backend": "ultraep",
181
+ "dynamic_nocg_status": "OK",
182
+ "dynamic_nocg_peak_gib": 100.2666015625,
183
+ "cap_1p13671875_status": "OK",
184
+ "cap_1p13671875_peak_gib": 99.6044921875
185
+ }
186
+ ]
187
+ }
granularity_next_b200/tierb_models/memory/DEEPSEEK_V3_L4/memory_summary_all.tsv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ run_dir config backend cuda_graph capacity_mode capacity status effective_peak_gib max_allocated_gib max_reserved_gib fragmentation_gib peak_sampled_gib snapshot_pickle_count snapshot_json_count snapshot_expected_each snapshot_complete branch commit
2
+ echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1 DEEPSEEK_V3_L4 echo 1 static 1.13671875 OK 128.0029296875 68.58609199523926 82.048828125 13.462736129760742 128.0029296875 48 48 48 1 better-memory-dump-support b7c52442
3
+ echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 DEEPSEEK_V3_L4 echo 0 dynamic dynamic OK 116.5556640625 69.96967601776123 79.791015625 9.82133960723877 116.5556640625 40 40 40 1 better-memory-dump-support b7c52442
4
+ moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1 DEEPSEEK_V3_L4 moonep 1 none none OK 95.5556640625 56.274383544921875 59.23046875 2.956085205078125 95.5556640625 48 48 48 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
5
+ moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1 DEEPSEEK_V3_L4 moonep 0 none none OK 81.8642578125 56.25963544845581 59.21484375 2.9552083015441895 81.8642578125 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
6
+ ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1 DEEPSEEK_V3_L4 ultraep 1 static 1.13671875 OK 99.6044921875 64.16005086898804 66.474609375 2.314558506011963 99.6044921875 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
7
+ ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 DEEPSEEK_V3_L4 ultraep 0 dynamic dynamic OK 100.2666015625 64.25602293014526 66.458984375 2.2029614448547363 100.2666015625 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7