AbhishekGhosh1998 commited on
Commit
1f61cf9
·
verified ·
1 Parent(s): 334ff7f

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T08:12:09Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 (part 2)

Browse files
tiera-rerun-fixed-moonep/hf_watcher.log CHANGED
@@ -58,3 +58,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
58
  Already logged in to Hugging Face as: AbhishekGhosh1998
59
  Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
60
  [2026-08-30T08:03:48+00:00] backup pass
 
 
 
58
  Already logged in to Hugging Face as: AbhishekGhosh1998
59
  Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
60
  [2026-08-30T08:03:48+00:00] backup pass
61
+ uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
62
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
tiera-rerun-fixed-moonep/memory_summary_all.json ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "runs": [
3
+ {
4
+ "run_dir": "echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1",
5
+ "config": "G1",
6
+ "backend": "echo",
7
+ "num_layers": "16",
8
+ "moe_layers": "16",
9
+ "seq_length": "4096",
10
+ "global_batch_size": "16",
11
+ "cuda_graph": "1",
12
+ "capacity_mode": "static",
13
+ "capacity": "2.40625",
14
+ "memory_dump_mode": "iteration",
15
+ "branch": "better-memory-dump-support",
16
+ "commit": "b7c52442",
17
+ "status": "OK",
18
+ "max_allocated_gib": 94.25749588012695,
19
+ "max_reserved_gib": 96.798828125,
20
+ "fragmentation_gib": 2.541332244873047,
21
+ "theoretical_weight_optim_gib": 165.067802734375,
22
+ "peak_sampled_gib": 157.9326171875,
23
+ "snapshot_pickle_count": 48,
24
+ "snapshot_json_count": 48,
25
+ "snapshot_expected_each": 48,
26
+ "snapshot_complete": 1,
27
+ "effective_peak_gib": 157.9326171875
28
+ },
29
+ {
30
+ "run_dir": "echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
31
+ "config": "G1",
32
+ "backend": "echo",
33
+ "num_layers": "16",
34
+ "moe_layers": "16",
35
+ "seq_length": "4096",
36
+ "global_batch_size": "16",
37
+ "cuda_graph": "0",
38
+ "capacity_mode": "dynamic",
39
+ "capacity": "dynamic",
40
+ "memory_dump_mode": "iteration",
41
+ "branch": "better-memory-dump-support",
42
+ "commit": "b7c52442",
43
+ "status": "OK",
44
+ "max_allocated_gib": 76.29637289047241,
45
+ "max_reserved_gib": 86.345703125,
46
+ "fragmentation_gib": 10.049330234527588,
47
+ "theoretical_weight_optim_gib": 165.067802734375,
48
+ "peak_sampled_gib": 112.5810546875,
49
+ "snapshot_pickle_count": 40,
50
+ "snapshot_json_count": 40,
51
+ "snapshot_expected_each": 40,
52
+ "snapshot_complete": 1,
53
+ "effective_peak_gib": 112.5810546875
54
+ },
55
+ {
56
+ "run_dir": "moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1",
57
+ "config": "G1",
58
+ "backend": "moonep",
59
+ "num_layers": "16",
60
+ "moe_layers": "16",
61
+ "seq_length": "4096",
62
+ "global_batch_size": "16",
63
+ "cuda_graph": "1",
64
+ "capacity_mode": "none",
65
+ "capacity": "none",
66
+ "memory_dump_mode": "iteration",
67
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
68
+ "commit": "7949f384",
69
+ "status": "OK",
70
+ "max_allocated_gib": 52.10025358200073,
71
+ "max_reserved_gib": 57.61328125,
72
+ "fragmentation_gib": 5.513027667999268,
73
+ "theoretical_weight_optim_gib": 165.067802734375,
74
+ "peak_sampled_gib": 94.5791015625,
75
+ "snapshot_pickle_count": 48,
76
+ "snapshot_json_count": 48,
77
+ "snapshot_expected_each": 48,
78
+ "snapshot_complete": 1,
79
+ "effective_peak_gib": 94.5791015625
80
+ },
81
+ {
82
+ "run_dir": "moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1",
83
+ "config": "G1",
84
+ "backend": "moonep",
85
+ "num_layers": "16",
86
+ "moe_layers": "16",
87
+ "seq_length": "4096",
88
+ "global_batch_size": "16",
89
+ "cuda_graph": "0",
90
+ "capacity_mode": "none",
91
+ "capacity": "none",
92
+ "memory_dump_mode": "iteration",
93
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
94
+ "commit": "7949f384",
95
+ "status": "OK",
96
+ "max_allocated_gib": 52.06590461730957,
97
+ "max_reserved_gib": 57.63671875,
98
+ "fragmentation_gib": 5.57081413269043,
99
+ "theoretical_weight_optim_gib": 165.067802734375,
100
+ "peak_sampled_gib": 90.4091796875,
101
+ "snapshot_pickle_count": 40,
102
+ "snapshot_json_count": 40,
103
+ "snapshot_expected_each": 40,
104
+ "snapshot_complete": 1,
105
+ "effective_peak_gib": 90.4091796875
106
+ },
107
+ {
108
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1",
109
+ "config": "G1",
110
+ "backend": "ultraep",
111
+ "num_layers": "16",
112
+ "moe_layers": "16",
113
+ "seq_length": "4096",
114
+ "global_batch_size": "16",
115
+ "cuda_graph": "1",
116
+ "capacity_mode": "static",
117
+ "capacity": "2.40625",
118
+ "memory_dump_mode": "iteration",
119
+ "branch": "ultraep-better-memory-dump-support",
120
+ "commit": "c3a8a8a7",
121
+ "status": "OK",
122
+ "max_allocated_gib": 83.64628076553345,
123
+ "max_reserved_gib": 85.63671875,
124
+ "fragmentation_gib": 1.9904379844665527,
125
+ "theoretical_weight_optim_gib": 165.067802734375,
126
+ "peak_sampled_gib": 126.4384765625,
127
+ "snapshot_pickle_count": 48,
128
+ "snapshot_json_count": 48,
129
+ "snapshot_expected_each": 48,
130
+ "snapshot_complete": 1,
131
+ "effective_peak_gib": 126.4384765625
132
+ },
133
+ {
134
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
135
+ "config": "G1",
136
+ "backend": "ultraep",
137
+ "num_layers": "16",
138
+ "moe_layers": "16",
139
+ "seq_length": "4096",
140
+ "global_batch_size": "16",
141
+ "cuda_graph": "0",
142
+ "capacity_mode": "dynamic",
143
+ "capacity": "dynamic",
144
+ "memory_dump_mode": "iteration",
145
+ "branch": "ultraep-better-memory-dump-support",
146
+ "commit": "c3a8a8a7",
147
+ "status": "OK",
148
+ "max_allocated_gib": 71.58890151977539,
149
+ "max_reserved_gib": 78.404296875,
150
+ "fragmentation_gib": 6.815395355224609,
151
+ "theoretical_weight_optim_gib": 165.067802734375,
152
+ "peak_sampled_gib": 106.7001953125,
153
+ "snapshot_pickle_count": 40,
154
+ "snapshot_json_count": 40,
155
+ "snapshot_expected_each": 40,
156
+ "snapshot_complete": 1,
157
+ "effective_peak_gib": 106.7001953125
158
+ }
159
+ ],
160
+ "comparisons": [
161
+ {
162
+ "config": "G1",
163
+ "backend": "echo",
164
+ "dynamic_nocg_status": "OK",
165
+ "dynamic_nocg_peak_gib": 112.5810546875,
166
+ "cap_2p40625_status": "OK",
167
+ "cap_2p40625_peak_gib": 157.9326171875
168
+ },
169
+ {
170
+ "config": "G1",
171
+ "backend": "moonep",
172
+ "moon_graph_on_status": "OK",
173
+ "moon_graph_on_peak_gib": 94.5791015625,
174
+ "moon_graph_off_status": "OK",
175
+ "moon_graph_off_peak_gib": 90.4091796875,
176
+ "moon_graph_delta_gib": 4.169921875
177
+ },
178
+ {
179
+ "config": "G1",
180
+ "backend": "ultraep",
181
+ "dynamic_nocg_status": "OK",
182
+ "dynamic_nocg_peak_gib": 106.7001953125,
183
+ "cap_2p40625_status": "OK",
184
+ "cap_2p40625_peak_gib": 126.4384765625
185
+ }
186
+ ]
187
+ }
tiera-rerun-fixed-moonep/memory_summary_all.tsv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ run_dir config backend cuda_graph capacity_mode capacity status effective_peak_gib max_allocated_gib max_reserved_gib fragmentation_gib peak_sampled_gib snapshot_pickle_count snapshot_json_count snapshot_expected_each snapshot_complete branch commit
2
+ echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1 G1 echo 1 static 2.40625 OK 157.9326171875 94.25749588012695 96.798828125 2.541332244873047 157.9326171875 48 48 48 1 better-memory-dump-support b7c52442
3
+ echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G1 echo 0 dynamic dynamic OK 112.5810546875 76.29637289047241 86.345703125 10.049330234527588 112.5810546875 40 40 40 1 better-memory-dump-support b7c52442
4
+ moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 G1 moonep 1 none none OK 94.5791015625 52.10025358200073 57.61328125 5.513027667999268 94.5791015625 48 48 48 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
5
+ moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G1 moonep 0 none none OK 90.4091796875 52.06590461730957 57.63671875 5.57081413269043 90.4091796875 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
6
+ ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1 G1 ultraep 1 static 2.40625 OK 126.4384765625 83.64628076553345 85.63671875 1.9904379844665527 126.4384765625 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
7
+ ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G1 ultraep 0 dynamic dynamic OK 106.7001953125 71.58890151977539 78.404296875 6.815395355224609 106.7001953125 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7