AbhishekGhosh1998 commited on
Commit
c35cd98
·
verified ·
1 Parent(s): c7610d5

granularity results echo/tierbmem @ 2026-08-26T19:10:38Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +248 -0
  2. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.json +79 -0
  3. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.pickle +3 -0
  4. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.json +79 -0
  5. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.pickle +3 -0
  6. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.json +79 -0
  7. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.pickle +3 -0
  8. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.json +79 -0
  9. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.pickle +3 -0
  10. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.json +79 -0
  11. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.pickle +3 -0
  12. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.json +79 -0
  13. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.pickle +3 -0
  14. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.json +79 -0
  15. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.pickle +3 -0
  16. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.json +79 -0
  17. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.pickle +3 -0
  18. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.json +79 -0
  19. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.pickle +3 -0
  20. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.json +79 -0
  21. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.pickle +3 -0
  22. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.json +79 -0
  23. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.pickle +3 -0
  24. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.json +79 -0
  25. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.pickle +3 -0
  26. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.json +79 -0
  27. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.pickle +3 -0
  28. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.json +79 -0
  29. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.pickle +3 -0
  30. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.json +79 -0
  31. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.pickle +3 -0
  32. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.json +79 -0
  33. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.pickle +3 -0
  34. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.json +79 -0
  35. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.pickle +3 -0
  36. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.json +79 -0
  37. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.pickle +3 -0
  38. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.json +79 -0
  39. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.pickle +3 -0
  40. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.json +79 -0
  41. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.pickle +3 -0
  42. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.json +79 -0
  43. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.pickle +3 -0
  44. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.json +79 -0
  45. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.pickle +3 -0
  46. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.json +79 -0
  47. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.pickle +3 -0
  48. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.json +79 -0
  49. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.pickle +3 -0
  50. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.json +79 -0
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 911
50
+ 1, 907
51
+ 2, 927
52
+ 3, 911
53
+ 4, 923
54
+ 5, 909
55
+ 6, 909
56
+ 7, 911
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 26289
66
+ 1, 26377
67
+ 2, 26223
68
+ 3, 26223
69
+ 4, 26223
70
+ 5, 26223
71
+ 6, 26355
72
+ 7, 26377
73
+ 0, 27983
74
+ 1, 27041
75
+ 2, 26889
76
+ 3, 26887
77
+ 4, 26887
78
+ 5, 27019
79
+ 6, 27409
80
+ 7, 27093
81
+ 0, 36841
82
+ 1, 36881
83
+ 2, 36701
84
+ 3, 36749
85
+ 4, 36773
86
+ 5, 36757
87
+ 6, 36919
88
+ 7, 36921
89
+ 0, 37785
90
+ 1, 37869
91
+ 2, 37719
92
+ 3, 37717
93
+ 4, 37717
94
+ 5, 37715
95
+ 6, 37851
96
+ 7, 37869
97
+ 0, 37873
98
+ 1, 37957
99
+ 2, 37807
100
+ 3, 37805
101
+ 4, 37805
102
+ 5, 37803
103
+ 6, 37939
104
+ 7, 37957
105
+ 0, 38079
106
+ 1, 38177
107
+ 2, 38011
108
+ 3, 37995
109
+ 4, 37991
110
+ 5, 37985
111
+ 6, 38129
112
+ 7, 38161
113
+ 0, 38821
114
+ 1, 38905
115
+ 2, 38755
116
+ 3, 38755
117
+ 4, 38755
118
+ 5, 38751
119
+ 6, 38887
120
+ 7, 38905
121
+ 0, 38821
122
+ 1, 38905
123
+ 2, 38755
124
+ 3, 38755
125
+ 4, 38755
126
+ 5, 38751
127
+ 6, 38887
128
+ 7, 38905
129
+ 0, 38827
130
+ 1, 38911
131
+ 2, 38761
132
+ 3, 38761
133
+ 4, 38761
134
+ 5, 38757
135
+ 6, 38893
136
+ 7, 38911
137
+ 0, 38829
138
+ 1, 38913
139
+ 2, 38763
140
+ 3, 38763
141
+ 4, 38763
142
+ 5, 38759
143
+ 6, 38895
144
+ 7, 38913
145
+ 0, 112013
146
+ 1, 112081
147
+ 2, 111981
148
+ 3, 111985
149
+ 4, 111961
150
+ 5, 111957
151
+ 6, 112053
152
+ 7, 112085
153
+ 0, 112443
154
+ 1, 112529
155
+ 2, 112377
156
+ 3, 112377
157
+ 4, 112377
158
+ 5, 112375
159
+ 6, 112509
160
+ 7, 112529
161
+ 0, 112443
162
+ 1, 112529
163
+ 2, 112377
164
+ 3, 112377
165
+ 4, 112377
166
+ 5, 112375
167
+ 6, 112509
168
+ 7, 112529
169
+ 0, 112443
170
+ 1, 112529
171
+ 2, 112377
172
+ 3, 112377
173
+ 4, 112377
174
+ 5, 112375
175
+ 6, 112509
176
+ 7, 112529
177
+ 0, 112443
178
+ 1, 112529
179
+ 2, 112377
180
+ 3, 112377
181
+ 4, 112377
182
+ 5, 112375
183
+ 6, 112509
184
+ 7, 112529
185
+ 0, 112973
186
+ 1, 113085
187
+ 2, 112929
188
+ 3, 112941
189
+ 4, 112893
190
+ 5, 112881
191
+ 6, 113007
192
+ 7, 113043
193
+ 0, 115965
194
+ 1, 116051
195
+ 2, 115899
196
+ 3, 115899
197
+ 4, 115899
198
+ 5, 115897
199
+ 6, 116031
200
+ 7, 116051
201
+ 0, 117031
202
+ 1, 117079
203
+ 2, 116949
204
+ 3, 116937
205
+ 4, 116891
206
+ 5, 116953
207
+ 6, 117053
208
+ 7, 117061
209
+ 0, 63497
210
+ 1, 63309
211
+ 2, 68747
212
+ 3, 63297
213
+ 4, 63269
214
+ 5, 63207
215
+ 6, 64683
216
+ 7, 63311
217
+ 0, 151283
218
+ 1, 151215
219
+ 2, 151229
220
+ 3, 151189
221
+ 4, 151189
222
+ 5, 151061
223
+ 6, 151261
224
+ 7, 151215
225
+ 0, 150337
226
+ 1, 150333
227
+ 2, 150929
228
+ 3, 150563
229
+ 4, 150307
230
+ 5, 150761
231
+ 6, 150635
232
+ 7, 150269
233
+ 0, 777
234
+ 1, 1034
235
+ 2, 150281
236
+ 3, 2058
237
+ 4, 778
238
+ 5, 778
239
+ 6, 778
240
+ 7, 778
241
+ 0, 0
242
+ 1, 0
243
+ 2, 0
244
+ 3, 0
245
+ 4, 0
246
+ 5, 0
247
+ 6, 0
248
+ 7, 0
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42487152128,
3
+ "active_bytes_current_delta": 19441586176,
4
+ "active_bytes_peak": 110640327680,
5
+ "active_bytes_peak_delta": 87582178816,
6
+ "after": {
7
+ "active_bytes_current": 42487152128,
8
+ "active_bytes_peak": 110640327680,
9
+ "allocated_bytes_current": 42487152128,
10
+ "allocated_bytes_peak": 110640327680,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 16369418752,
13
+ "inactive_split_bytes_peak": 28209760256,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 115332874240,
21
+ "reserved_bytes_peak": 115332874240,
22
+ "segment_count_current": 922,
23
+ "segment_count_peak": 922,
24
+ "timestamp_utc": "2026-08-26T19:01:23.031427Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42487152128,
28
+ "allocated_bytes_current_delta": 19441586176,
29
+ "allocated_bytes_peak": 110640327680,
30
+ "allocated_bytes_peak_delta": 87582178816,
31
+ "before": {
32
+ "active_bytes_current": 23045565952,
33
+ "active_bytes_peak": 23058148864,
34
+ "allocated_bytes_current": 23045565952,
35
+ "allocated_bytes_peak": 23058148864,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 352358912,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 25692209152,
46
+ "reserved_bytes_peak": 25692209152,
47
+ "segment_count_current": 554,
48
+ "segment_count_peak": 554,
49
+ "timestamp_utc": "2026-08-26T19:00:42.978810Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16369418752,
55
+ "inactive_split_bytes_current_delta": 16017059840,
56
+ "inactive_split_bytes_peak": 28209760256,
57
+ "inactive_split_bytes_peak_delta": 27326524928,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115332874240,
68
+ "reserved_bytes_current_delta": 89640665088,
69
+ "reserved_bytes_peak": 115332874240,
70
+ "reserved_bytes_peak_delta": 89640665088,
71
+ "segment_count_current": 922,
72
+ "segment_count_current_delta": 368,
73
+ "segment_count_peak": 922,
74
+ "segment_count_peak_delta": 368,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.114919Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adac9585809518aeee8b590eb1dbac1b7b475832d71017a666d17a26187ae06c
3
+ size 4203011
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42487152128,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 124994119680,
5
+ "active_bytes_peak_delta": 14353792000,
6
+ "after": {
7
+ "active_bytes_current": 42487152128,
8
+ "active_bytes_peak": 124994119680,
9
+ "allocated_bytes_current": 42487152128,
10
+ "allocated_bytes_peak": 124855707648,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 16461693440,
13
+ "inactive_split_bytes_peak": 28547580416,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 127729139712,
21
+ "reserved_bytes_peak": 127729139712,
22
+ "segment_count_current": 933,
23
+ "segment_count_peak": 933,
24
+ "timestamp_utc": "2026-08-26T19:01:23.760912Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42487152128,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124855707648,
30
+ "allocated_bytes_peak_delta": 14215379968,
31
+ "before": {
32
+ "active_bytes_current": 42487152128,
33
+ "active_bytes_peak": 110640327680,
34
+ "allocated_bytes_current": 42487152128,
35
+ "allocated_bytes_peak": 110640327680,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 16369418752,
38
+ "inactive_split_bytes_peak": 28209760256,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 115332874240,
46
+ "reserved_bytes_peak": 115332874240,
47
+ "segment_count_current": 922,
48
+ "segment_count_peak": 922,
49
+ "timestamp_utc": "2026-08-26T19:01:23.127117Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16461693440,
55
+ "inactive_split_bytes_current_delta": 92274688,
56
+ "inactive_split_bytes_peak": 28547580416,
57
+ "inactive_split_bytes_peak_delta": 337820160,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127729139712,
68
+ "reserved_bytes_current_delta": 12396265472,
69
+ "reserved_bytes_peak": 127729139712,
70
+ "reserved_bytes_peak_delta": 12396265472,
71
+ "segment_count_current": 933,
72
+ "segment_count_current_delta": 11,
73
+ "segment_count_peak": 933,
74
+ "segment_count_peak_delta": 11,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.838750Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:695bf7566f0ca55c666db0475079c916bca930ca2e25b6aba6f827de5871eb47
3
+ size 4587773
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42487152128,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125133957120,
5
+ "active_bytes_peak_delta": 139837440,
6
+ "after": {
7
+ "active_bytes_current": 42487152128,
8
+ "active_bytes_peak": 125133957120,
9
+ "allocated_bytes_current": 42487152128,
10
+ "allocated_bytes_peak": 124857133056,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 16461693440,
13
+ "inactive_split_bytes_peak": 29876894208,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 127729139712,
21
+ "reserved_bytes_peak": 127729139712,
22
+ "segment_count_current": 933,
23
+ "segment_count_peak": 933,
24
+ "timestamp_utc": "2026-08-26T19:01:24.484602Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42487152128,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124857133056,
30
+ "allocated_bytes_peak_delta": 1425408,
31
+ "before": {
32
+ "active_bytes_current": 42487152128,
33
+ "active_bytes_peak": 124994119680,
34
+ "allocated_bytes_current": 42487152128,
35
+ "allocated_bytes_peak": 124855707648,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 16461693440,
38
+ "inactive_split_bytes_peak": 28547580416,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 127729139712,
46
+ "reserved_bytes_peak": 127729139712,
47
+ "segment_count_current": 933,
48
+ "segment_count_peak": 933,
49
+ "timestamp_utc": "2026-08-26T19:01:23.848707Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16461693440,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 29876894208,
57
+ "inactive_split_bytes_peak_delta": 1329313792,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127729139712,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 127729139712,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 933,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 933,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:24.559979Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e91880282c9511ee39ff513da9b5592529c634c7bc780ecb33462c24b71b5f1d
3
+ size 4605081
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42505113088,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 127972926464,
5
+ "active_bytes_peak_delta": 2838969344,
6
+ "after": {
7
+ "active_bytes_current": 42505113088,
8
+ "active_bytes_peak": 127972926464,
9
+ "allocated_bytes_current": 42505113088,
10
+ "allocated_bytes_peak": 124857133056,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 18522010112,
13
+ "inactive_split_bytes_peak": 43042810368,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 151074635776,
21
+ "reserved_bytes_peak": 151074635776,
22
+ "segment_count_current": 1130,
23
+ "segment_count_peak": 1130,
24
+ "timestamp_utc": "2026-08-26T19:01:26.509405Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42505113088,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 124857133056,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42487152128,
33
+ "active_bytes_peak": 125133957120,
34
+ "allocated_bytes_current": 42487152128,
35
+ "allocated_bytes_peak": 124857133056,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 16461693440,
38
+ "inactive_split_bytes_peak": 29876894208,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 127729139712,
46
+ "reserved_bytes_peak": 127729139712,
47
+ "segment_count_current": 933,
48
+ "segment_count_peak": 933,
49
+ "timestamp_utc": "2026-08-26T19:01:24.569517Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18522010112,
55
+ "inactive_split_bytes_current_delta": 2060316672,
56
+ "inactive_split_bytes_peak": 43042810368,
57
+ "inactive_split_bytes_peak_delta": 13165916160,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151074635776,
68
+ "reserved_bytes_current_delta": 23345496064,
69
+ "reserved_bytes_peak": 151074635776,
70
+ "reserved_bytes_peak_delta": 23345496064,
71
+ "segment_count_current": 1130,
72
+ "segment_count_current_delta": 197,
73
+ "segment_count_peak": 1130,
74
+ "segment_count_peak_delta": 197,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:26.596160Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07ffd7cfe481a42d34cd7fbc78f6abb67a51083535c6b5ce422c0b0e07b4576e
3
+ size 4739794
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42505113088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127972926464,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42505113088,
8
+ "active_bytes_peak": 127972926464,
9
+ "allocated_bytes_current": 42505113088,
10
+ "allocated_bytes_peak": 124857133056,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 18522010112,
13
+ "inactive_split_bytes_peak": 43042810368,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 151074635776,
21
+ "reserved_bytes_peak": 151074635776,
22
+ "segment_count_current": 1130,
23
+ "segment_count_peak": 1130,
24
+ "timestamp_utc": "2026-08-26T19:01:27.191302Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42505113088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124857133056,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42505113088,
33
+ "active_bytes_peak": 127972926464,
34
+ "allocated_bytes_current": 42505113088,
35
+ "allocated_bytes_peak": 124857133056,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 18522010112,
38
+ "inactive_split_bytes_peak": 43042810368,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 151074635776,
46
+ "reserved_bytes_peak": 151074635776,
47
+ "segment_count_current": 1130,
48
+ "segment_count_peak": 1130,
49
+ "timestamp_utc": "2026-08-26T19:01:26.606920Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18522010112,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 43042810368,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151074635776,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 151074635776,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1130,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1130,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.201978Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0bfa5cbc889b82e95faa85d9e523b727238f1706c44b32b7ac1b134ddd33f250
3
+ size 383472
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42505113088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127972926464,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42505113088,
8
+ "active_bytes_peak": 127972926464,
9
+ "allocated_bytes_current": 42505113088,
10
+ "allocated_bytes_peak": 124857133056,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 18522010112,
13
+ "inactive_split_bytes_peak": 43042810368,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 151074635776,
21
+ "reserved_bytes_peak": 151074635776,
22
+ "segment_count_current": 1130,
23
+ "segment_count_peak": 1130,
24
+ "timestamp_utc": "2026-08-26T19:01:27.792453Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42505113088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124857133056,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42505113088,
33
+ "active_bytes_peak": 127972926464,
34
+ "allocated_bytes_current": 42505113088,
35
+ "allocated_bytes_peak": 124857133056,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 18522010112,
38
+ "inactive_split_bytes_peak": 43042810368,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 151074635776,
46
+ "reserved_bytes_peak": 151074635776,
47
+ "segment_count_current": 1130,
48
+ "segment_count_peak": 1130,
49
+ "timestamp_utc": "2026-08-26T19:01:27.203084Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18522010112,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 43042810368,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151074635776,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 151074635776,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1130,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1130,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.802293Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0fce639d03a24fcadde45bffc3e5c620abacd21544d6729772208d908860dc2
3
+ size 383472
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42495456768,
3
+ "active_bytes_current_delta": 19447151104,
4
+ "active_bytes_peak": 110662637568,
5
+ "active_bytes_peak_delta": 87601748992,
6
+ "after": {
7
+ "active_bytes_current": 42495456768,
8
+ "active_bytes_peak": 110662637568,
9
+ "allocated_bytes_current": 42495456768,
10
+ "allocated_bytes_peak": 110662637568,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 16382085632,
13
+ "inactive_split_bytes_peak": 28267021312,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 115423051776,
21
+ "reserved_bytes_peak": 115423051776,
22
+ "segment_count_current": 925,
23
+ "segment_count_peak": 925,
24
+ "timestamp_utc": "2026-08-26T19:01:23.031572Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42495456768,
28
+ "allocated_bytes_current_delta": 19447151104,
29
+ "allocated_bytes_peak": 110662637568,
30
+ "allocated_bytes_peak_delta": 87601748992,
31
+ "before": {
32
+ "active_bytes_current": 23048305664,
33
+ "active_bytes_peak": 23060888576,
34
+ "allocated_bytes_current": 23048305664,
35
+ "allocated_bytes_peak": 23060888576,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 502711296,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 25784483840,
46
+ "reserved_bytes_peak": 25784483840,
47
+ "segment_count_current": 558,
48
+ "segment_count_peak": 558,
49
+ "timestamp_utc": "2026-08-26T19:00:42.978679Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16382085632,
55
+ "inactive_split_bytes_current_delta": 15879374336,
56
+ "inactive_split_bytes_peak": 28267021312,
57
+ "inactive_split_bytes_peak_delta": 27383785984,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115423051776,
68
+ "reserved_bytes_current_delta": 89638567936,
69
+ "reserved_bytes_peak": 115423051776,
70
+ "reserved_bytes_peak_delta": 89638567936,
71
+ "segment_count_current": 925,
72
+ "segment_count_current_delta": 367,
73
+ "segment_count_peak": 925,
74
+ "segment_count_peak_delta": 367,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.113479Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae1e8a691184138cf45ed9c4cfd257c5a59a7cfc0efa87846d81383e41ac8205
3
+ size 4145363
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42495456768,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 124990884864,
5
+ "active_bytes_peak_delta": 14328247296,
6
+ "after": {
7
+ "active_bytes_current": 42495456768,
8
+ "active_bytes_peak": 124990884864,
9
+ "allocated_bytes_current": 42495456768,
10
+ "allocated_bytes_peak": 124852472832,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 16474360320,
13
+ "inactive_split_bytes_peak": 28629453312,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 127815122944,
21
+ "reserved_bytes_peak": 127815122944,
22
+ "segment_count_current": 934,
23
+ "segment_count_peak": 934,
24
+ "timestamp_utc": "2026-08-26T19:01:23.760843Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42495456768,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124852472832,
30
+ "allocated_bytes_peak_delta": 14189835264,
31
+ "before": {
32
+ "active_bytes_current": 42495456768,
33
+ "active_bytes_peak": 110662637568,
34
+ "allocated_bytes_current": 42495456768,
35
+ "allocated_bytes_peak": 110662637568,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 16382085632,
38
+ "inactive_split_bytes_peak": 28267021312,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 115423051776,
46
+ "reserved_bytes_peak": 115423051776,
47
+ "segment_count_current": 925,
48
+ "segment_count_peak": 925,
49
+ "timestamp_utc": "2026-08-26T19:01:23.124351Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16474360320,
55
+ "inactive_split_bytes_current_delta": 92274688,
56
+ "inactive_split_bytes_peak": 28629453312,
57
+ "inactive_split_bytes_peak_delta": 362432000,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127815122944,
68
+ "reserved_bytes_current_delta": 12392071168,
69
+ "reserved_bytes_peak": 127815122944,
70
+ "reserved_bytes_peak_delta": 12392071168,
71
+ "segment_count_current": 934,
72
+ "segment_count_current_delta": 9,
73
+ "segment_count_peak": 934,
74
+ "segment_count_peak_delta": 9,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.834929Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a75fa12914093db306eb848547efb9a8b75549cbf2f17fbe222156468bf99657
3
+ size 4529532
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42495456768,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125128502272,
5
+ "active_bytes_peak_delta": 137617408,
6
+ "after": {
7
+ "active_bytes_current": 42495456768,
8
+ "active_bytes_peak": 125128502272,
9
+ "allocated_bytes_current": 42495456768,
10
+ "allocated_bytes_peak": 124852472832,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 16382085632,
13
+ "inactive_split_bytes_peak": 28629453312,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 127815122944,
21
+ "reserved_bytes_peak": 127815122944,
22
+ "segment_count_current": 934,
23
+ "segment_count_peak": 934,
24
+ "timestamp_utc": "2026-08-26T19:01:24.484674Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42495456768,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124852472832,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42495456768,
33
+ "active_bytes_peak": 124990884864,
34
+ "allocated_bytes_current": 42495456768,
35
+ "allocated_bytes_peak": 124852472832,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 16474360320,
38
+ "inactive_split_bytes_peak": 28629453312,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 127815122944,
46
+ "reserved_bytes_peak": 127815122944,
47
+ "segment_count_current": 934,
48
+ "segment_count_peak": 934,
49
+ "timestamp_utc": "2026-08-26T19:01:23.844050Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16382085632,
55
+ "inactive_split_bytes_current_delta": -92274688,
56
+ "inactive_split_bytes_peak": 28629453312,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127815122944,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 127815122944,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 934,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:24.558894Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7c57e637480c1be5c13129517958afc9cf9edf24f6a11e1f8a0ba7ff2193256
3
+ size 4546981
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42513417728,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 127981231104,
5
+ "active_bytes_peak_delta": 2852728832,
6
+ "after": {
7
+ "active_bytes_current": 42513417728,
8
+ "active_bytes_peak": 127981231104,
9
+ "allocated_bytes_current": 42513417728,
10
+ "allocated_bytes_peak": 124852472832,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 18534676992,
13
+ "inactive_split_bytes_peak": 42956911104,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 151003332608,
21
+ "reserved_bytes_peak": 151003332608,
22
+ "segment_count_current": 1099,
23
+ "segment_count_peak": 1099,
24
+ "timestamp_utc": "2026-08-26T19:01:26.509381Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42513417728,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 124852472832,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42495456768,
33
+ "active_bytes_peak": 125128502272,
34
+ "allocated_bytes_current": 42495456768,
35
+ "allocated_bytes_peak": 124852472832,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 16382085632,
38
+ "inactive_split_bytes_peak": 28629453312,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 127815122944,
46
+ "reserved_bytes_peak": 127815122944,
47
+ "segment_count_current": 934,
48
+ "segment_count_peak": 934,
49
+ "timestamp_utc": "2026-08-26T19:01:24.567614Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18534676992,
55
+ "inactive_split_bytes_current_delta": 2152591360,
56
+ "inactive_split_bytes_peak": 42956911104,
57
+ "inactive_split_bytes_peak_delta": 14327457792,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151003332608,
68
+ "reserved_bytes_current_delta": 23188209664,
69
+ "reserved_bytes_peak": 151003332608,
70
+ "reserved_bytes_peak_delta": 23188209664,
71
+ "segment_count_current": 1099,
72
+ "segment_count_current_delta": 165,
73
+ "segment_count_peak": 1099,
74
+ "segment_count_peak_delta": 165,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:26.594279Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59eaaf79332db41fce0df031c651253d496b57ff4f14951a4684376391ea3934
3
+ size 4764032
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42513417728,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127981231104,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42513417728,
8
+ "active_bytes_peak": 127981231104,
9
+ "allocated_bytes_current": 42513417728,
10
+ "allocated_bytes_peak": 124852472832,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 18534676992,
13
+ "inactive_split_bytes_peak": 42956911104,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 151003332608,
21
+ "reserved_bytes_peak": 151003332608,
22
+ "segment_count_current": 1099,
23
+ "segment_count_peak": 1099,
24
+ "timestamp_utc": "2026-08-26T19:01:27.191308Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42513417728,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124852472832,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42513417728,
33
+ "active_bytes_peak": 127981231104,
34
+ "allocated_bytes_current": 42513417728,
35
+ "allocated_bytes_peak": 124852472832,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 18534676992,
38
+ "inactive_split_bytes_peak": 42956911104,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 151003332608,
46
+ "reserved_bytes_peak": 151003332608,
47
+ "segment_count_current": 1099,
48
+ "segment_count_peak": 1099,
49
+ "timestamp_utc": "2026-08-26T19:01:26.604257Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18534676992,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 42956911104,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151003332608,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 151003332608,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1099,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1099,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.201474Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c1e11ca04bb16997c2d624984b6ace9dc226f4ae5482e86409ddc39b3e32e38
3
+ size 405776
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42513417728,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127981231104,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42513417728,
8
+ "active_bytes_peak": 127981231104,
9
+ "allocated_bytes_current": 42513417728,
10
+ "allocated_bytes_peak": 124852472832,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 18534676992,
13
+ "inactive_split_bytes_peak": 42956911104,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 151003332608,
21
+ "reserved_bytes_peak": 151003332608,
22
+ "segment_count_current": 1099,
23
+ "segment_count_peak": 1099,
24
+ "timestamp_utc": "2026-08-26T19:01:27.792496Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42513417728,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124852472832,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42513417728,
33
+ "active_bytes_peak": 127981231104,
34
+ "allocated_bytes_current": 42513417728,
35
+ "allocated_bytes_peak": 124852472832,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 18534676992,
38
+ "inactive_split_bytes_peak": 42956911104,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 151003332608,
46
+ "reserved_bytes_peak": 151003332608,
47
+ "segment_count_current": 1099,
48
+ "segment_count_peak": 1099,
49
+ "timestamp_utc": "2026-08-26T19:01:27.202606Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18534676992,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 42956911104,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151003332608,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 151003332608,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1099,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1099,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.802691Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f190610ca87db8b2535504aa7ad66a6c3aa4957dd522af68693ad66b4202f32
3
+ size 405776
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42478725120,
3
+ "active_bytes_current_delta": 19427855360,
4
+ "active_bytes_peak": 110641677312,
5
+ "active_bytes_peak_delta": 87578224640,
6
+ "after": {
7
+ "active_bytes_current": 42478725120,
8
+ "active_bytes_peak": 110641677312,
9
+ "allocated_bytes_current": 42478725120,
10
+ "allocated_bytes_peak": 110641677312,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 16319125504,
13
+ "inactive_split_bytes_peak": 28242721792,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 115263668224,
21
+ "reserved_bytes_peak": 115263668224,
22
+ "segment_count_current": 919,
23
+ "segment_count_peak": 919,
24
+ "timestamp_utc": "2026-08-26T19:01:23.031410Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42478725120,
28
+ "allocated_bytes_current_delta": 19427855360,
29
+ "allocated_bytes_peak": 110641677312,
30
+ "allocated_bytes_peak_delta": 87578224640,
31
+ "before": {
32
+ "active_bytes_current": 23050869760,
33
+ "active_bytes_peak": 23063452672,
34
+ "allocated_bytes_current": 23050869760,
35
+ "allocated_bytes_peak": 23063452672,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 363832320,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 25623003136,
46
+ "reserved_bytes_peak": 25623003136,
47
+ "segment_count_current": 551,
48
+ "segment_count_peak": 551,
49
+ "timestamp_utc": "2026-08-26T19:00:42.978802Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16319125504,
55
+ "inactive_split_bytes_current_delta": 15955293184,
56
+ "inactive_split_bytes_peak": 28242721792,
57
+ "inactive_split_bytes_peak_delta": 27359486464,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115263668224,
68
+ "reserved_bytes_current_delta": 89640665088,
69
+ "reserved_bytes_peak": 115263668224,
70
+ "reserved_bytes_peak_delta": 89640665088,
71
+ "segment_count_current": 919,
72
+ "segment_count_current_delta": 368,
73
+ "segment_count_peak": 919,
74
+ "segment_count_peak_delta": 368,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.115545Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e86fb2ca11ce46c42d117498573ab954f8e0ba6596e1ede0faef8c4c1998b51
3
+ size 4127102
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42478725120,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 124971887616,
5
+ "active_bytes_peak_delta": 14330210304,
6
+ "after": {
7
+ "active_bytes_current": 42478725120,
8
+ "active_bytes_peak": 124971887616,
9
+ "allocated_bytes_current": 42478725120,
10
+ "allocated_bytes_peak": 124833475584,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 16411400192,
13
+ "inactive_split_bytes_peak": 28242721792,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 127662030848,
21
+ "reserved_bytes_peak": 127662030848,
22
+ "segment_count_current": 931,
23
+ "segment_count_peak": 931,
24
+ "timestamp_utc": "2026-08-26T19:01:23.760848Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42478725120,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124833475584,
30
+ "allocated_bytes_peak_delta": 14191798272,
31
+ "before": {
32
+ "active_bytes_current": 42478725120,
33
+ "active_bytes_peak": 110641677312,
34
+ "allocated_bytes_current": 42478725120,
35
+ "allocated_bytes_peak": 110641677312,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 16319125504,
38
+ "inactive_split_bytes_peak": 28242721792,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 115263668224,
46
+ "reserved_bytes_peak": 115263668224,
47
+ "segment_count_current": 919,
48
+ "segment_count_peak": 919,
49
+ "timestamp_utc": "2026-08-26T19:01:23.127162Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16411400192,
55
+ "inactive_split_bytes_current_delta": 92274688,
56
+ "inactive_split_bytes_peak": 28242721792,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127662030848,
68
+ "reserved_bytes_current_delta": 12398362624,
69
+ "reserved_bytes_peak": 127662030848,
70
+ "reserved_bytes_peak_delta": 12398362624,
71
+ "segment_count_current": 931,
72
+ "segment_count_current_delta": 12,
73
+ "segment_count_peak": 931,
74
+ "segment_count_peak_delta": 12,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.837243Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4e3a7ba70cad4ed95258c7610b044f2cdf5c9676e70ba3c3ff81a7ffcd3f8a3
3
+ size 4509234
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42478725120,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125107940864,
5
+ "active_bytes_peak_delta": 136053248,
6
+ "after": {
7
+ "active_bytes_current": 42478725120,
8
+ "active_bytes_peak": 125107940864,
9
+ "allocated_bytes_current": 42478725120,
10
+ "allocated_bytes_peak": 124833475584,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 16413497344,
13
+ "inactive_split_bytes_peak": 28242721792,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 127754305536,
21
+ "reserved_bytes_peak": 127754305536,
22
+ "segment_count_current": 932,
23
+ "segment_count_peak": 932,
24
+ "timestamp_utc": "2026-08-26T19:01:24.484587Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42478725120,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124833475584,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42478725120,
33
+ "active_bytes_peak": 124971887616,
34
+ "allocated_bytes_current": 42478725120,
35
+ "allocated_bytes_peak": 124833475584,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 16411400192,
38
+ "inactive_split_bytes_peak": 28242721792,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 127662030848,
46
+ "reserved_bytes_peak": 127662030848,
47
+ "segment_count_current": 931,
48
+ "segment_count_peak": 931,
49
+ "timestamp_utc": "2026-08-26T19:01:23.846392Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16413497344,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 28242721792,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127754305536,
68
+ "reserved_bytes_current_delta": 92274688,
69
+ "reserved_bytes_peak": 127754305536,
70
+ "reserved_bytes_peak_delta": 92274688,
71
+ "segment_count_current": 932,
72
+ "segment_count_current_delta": 1,
73
+ "segment_count_peak": 932,
74
+ "segment_count_peak_delta": 1,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:24.559130Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:383a5aee6269d371c8d80d2e95f9992e08158c80891087fa1ed8936cf99b6170
3
+ size 4526520
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42496686080,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 127964499456,
5
+ "active_bytes_peak_delta": 2856558592,
6
+ "after": {
7
+ "active_bytes_current": 42496686080,
8
+ "active_bytes_peak": 127964499456,
9
+ "allocated_bytes_current": 42496686080,
10
+ "allocated_bytes_peak": 124833475584,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 18475911168,
13
+ "inactive_split_bytes_peak": 42998808576,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 151018012672,
21
+ "reserved_bytes_peak": 151018012672,
22
+ "segment_count_current": 1133,
23
+ "segment_count_peak": 1133,
24
+ "timestamp_utc": "2026-08-26T19:01:26.509398Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42496686080,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 124833475584,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42478725120,
33
+ "active_bytes_peak": 125107940864,
34
+ "allocated_bytes_current": 42478725120,
35
+ "allocated_bytes_peak": 124833475584,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 16413497344,
38
+ "inactive_split_bytes_peak": 28242721792,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 127754305536,
46
+ "reserved_bytes_peak": 127754305536,
47
+ "segment_count_current": 932,
48
+ "segment_count_peak": 932,
49
+ "timestamp_utc": "2026-08-26T19:01:24.567968Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18475911168,
55
+ "inactive_split_bytes_current_delta": 2062413824,
56
+ "inactive_split_bytes_peak": 42998808576,
57
+ "inactive_split_bytes_peak_delta": 14756086784,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151018012672,
68
+ "reserved_bytes_current_delta": 23263707136,
69
+ "reserved_bytes_peak": 151018012672,
70
+ "reserved_bytes_peak_delta": 23263707136,
71
+ "segment_count_current": 1133,
72
+ "segment_count_current_delta": 201,
73
+ "segment_count_peak": 1133,
74
+ "segment_count_peak_delta": 201,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:26.595370Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8abaa17e2a9bd1c4298e1c241b78780bebb5b507fc866cf26c32c3cc4196c8a5
3
+ size 4737545
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42496686080,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127964499456,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42496686080,
8
+ "active_bytes_peak": 127964499456,
9
+ "allocated_bytes_current": 42496686080,
10
+ "allocated_bytes_peak": 124833475584,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 18475911168,
13
+ "inactive_split_bytes_peak": 42998808576,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 151018012672,
21
+ "reserved_bytes_peak": 151018012672,
22
+ "segment_count_current": 1133,
23
+ "segment_count_peak": 1133,
24
+ "timestamp_utc": "2026-08-26T19:01:27.191302Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42496686080,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124833475584,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42496686080,
33
+ "active_bytes_peak": 127964499456,
34
+ "allocated_bytes_current": 42496686080,
35
+ "allocated_bytes_peak": 124833475584,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 18475911168,
38
+ "inactive_split_bytes_peak": 42998808576,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 151018012672,
46
+ "reserved_bytes_peak": 151018012672,
47
+ "segment_count_current": 1133,
48
+ "segment_count_peak": 1133,
49
+ "timestamp_utc": "2026-08-26T19:01:26.605464Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18475911168,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 42998808576,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151018012672,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 151018012672,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1133,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1133,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.201722Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1450544ece0e239156314d91696b8be88c18a449024e0ee7ed120fe2ae44a393
3
+ size 382821
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42496686080,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127964499456,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42496686080,
8
+ "active_bytes_peak": 127964499456,
9
+ "allocated_bytes_current": 42496686080,
10
+ "allocated_bytes_peak": 124833475584,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 18475911168,
13
+ "inactive_split_bytes_peak": 42998808576,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 151018012672,
21
+ "reserved_bytes_peak": 151018012672,
22
+ "segment_count_current": 1133,
23
+ "segment_count_peak": 1133,
24
+ "timestamp_utc": "2026-08-26T19:01:27.792458Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42496686080,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124833475584,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42496686080,
33
+ "active_bytes_peak": 127964499456,
34
+ "allocated_bytes_current": 42496686080,
35
+ "allocated_bytes_peak": 124833475584,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 18475911168,
38
+ "inactive_split_bytes_peak": 42998808576,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 151018012672,
46
+ "reserved_bytes_peak": 151018012672,
47
+ "segment_count_current": 1133,
48
+ "segment_count_peak": 1133,
49
+ "timestamp_utc": "2026-08-26T19:01:27.202836Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18475911168,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 42998808576,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 151018012672,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 151018012672,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1133,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1133,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.802004Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:166f9a52a0c266d251cb976688afc69a55d1e67de9721b7d5b8b76c4284f6a93
3
+ size 382821
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42492211712,
3
+ "active_bytes_current_delta": 19443268608,
4
+ "active_bytes_peak": 110661306368,
5
+ "active_bytes_peak_delta": 87599780352,
6
+ "after": {
7
+ "active_bytes_current": 42492211712,
8
+ "active_bytes_peak": 110661306368,
9
+ "allocated_bytes_current": 42492211712,
10
+ "allocated_bytes_peak": 110661306368,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 16263695872,
13
+ "inactive_split_bytes_peak": 28274912256,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 115263668224,
21
+ "reserved_bytes_peak": 115263668224,
22
+ "segment_count_current": 919,
23
+ "segment_count_peak": 919,
24
+ "timestamp_utc": "2026-08-26T19:01:23.031470Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42492211712,
28
+ "allocated_bytes_current_delta": 19443268608,
29
+ "allocated_bytes_peak": 110661306368,
30
+ "allocated_bytes_peak_delta": 87599780352,
31
+ "before": {
32
+ "active_bytes_current": 23048943104,
33
+ "active_bytes_peak": 23061526016,
34
+ "allocated_bytes_current": 23048943104,
35
+ "allocated_bytes_peak": 23061526016,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 468519424,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 25623003136,
46
+ "reserved_bytes_peak": 25623003136,
47
+ "segment_count_current": 551,
48
+ "segment_count_peak": 551,
49
+ "timestamp_utc": "2026-08-26T19:00:42.978699Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16263695872,
55
+ "inactive_split_bytes_current_delta": 15795176448,
56
+ "inactive_split_bytes_peak": 28274912256,
57
+ "inactive_split_bytes_peak_delta": 27391676928,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115263668224,
68
+ "reserved_bytes_current_delta": 89640665088,
69
+ "reserved_bytes_peak": 115263668224,
70
+ "reserved_bytes_peak_delta": 89640665088,
71
+ "segment_count_current": 919,
72
+ "segment_count_current_delta": 368,
73
+ "segment_count_peak": 919,
74
+ "segment_count_peak_delta": 368,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.114264Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:caeb44b6301ce1a7c98ad82eb2306e870843f0c4a8305c956f996fa8bbf84aa0
3
+ size 4167516
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42492211712,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 124978370560,
5
+ "active_bytes_peak_delta": 14317064192,
6
+ "after": {
7
+ "active_bytes_current": 42492211712,
8
+ "active_bytes_peak": 124978370560,
9
+ "allocated_bytes_current": 42492211712,
10
+ "allocated_bytes_peak": 124839958528,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 16358067712,
13
+ "inactive_split_bytes_peak": 28274912256,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 129805320192,
21
+ "reserved_bytes_peak": 129805320192,
22
+ "segment_count_current": 930,
23
+ "segment_count_peak": 930,
24
+ "timestamp_utc": "2026-08-26T19:01:23.760842Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42492211712,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124839958528,
30
+ "allocated_bytes_peak_delta": 14178652160,
31
+ "before": {
32
+ "active_bytes_current": 42492211712,
33
+ "active_bytes_peak": 110661306368,
34
+ "allocated_bytes_current": 42492211712,
35
+ "allocated_bytes_peak": 110661306368,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 16263695872,
38
+ "inactive_split_bytes_peak": 28274912256,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 115263668224,
46
+ "reserved_bytes_peak": 115263668224,
47
+ "segment_count_current": 919,
48
+ "segment_count_peak": 919,
49
+ "timestamp_utc": "2026-08-26T19:01:23.125881Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16358067712,
55
+ "inactive_split_bytes_current_delta": 94371840,
56
+ "inactive_split_bytes_peak": 28274912256,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 129805320192,
68
+ "reserved_bytes_current_delta": 14541651968,
69
+ "reserved_bytes_peak": 129805320192,
70
+ "reserved_bytes_peak_delta": 14541651968,
71
+ "segment_count_current": 930,
72
+ "segment_count_current_delta": 11,
73
+ "segment_count_peak": 930,
74
+ "segment_count_peak_delta": 11,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.836274Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:921102b11c2112f3c18c026a0dd9768cf606509d3076f194158dfc02249ac88a
3
+ size 4585585
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42492211712,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125118871552,
5
+ "active_bytes_peak_delta": 140500992,
6
+ "after": {
7
+ "active_bytes_current": 42492211712,
8
+ "active_bytes_peak": 125118871552,
9
+ "allocated_bytes_current": 42492211712,
10
+ "allocated_bytes_peak": 124842047488,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 16358067712,
13
+ "inactive_split_bytes_peak": 28274912256,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 129805320192,
21
+ "reserved_bytes_peak": 129805320192,
22
+ "segment_count_current": 930,
23
+ "segment_count_peak": 930,
24
+ "timestamp_utc": "2026-08-26T19:01:24.484592Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42492211712,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124842047488,
30
+ "allocated_bytes_peak_delta": 2088960,
31
+ "before": {
32
+ "active_bytes_current": 42492211712,
33
+ "active_bytes_peak": 124978370560,
34
+ "allocated_bytes_current": 42492211712,
35
+ "allocated_bytes_peak": 124839958528,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 16358067712,
38
+ "inactive_split_bytes_peak": 28274912256,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 129805320192,
46
+ "reserved_bytes_peak": 129805320192,
47
+ "segment_count_current": 930,
48
+ "segment_count_peak": 930,
49
+ "timestamp_utc": "2026-08-26T19:01:23.845623Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16358067712,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 28274912256,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 129805320192,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 129805320192,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 930,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 930,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:24.560189Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f433d080b92a226a5558d1c2b013896ff57a42e10767bbc0b0e0b106cf54458b
3
+ size 4572603
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42510172672,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 127977986048,
5
+ "active_bytes_peak_delta": 2859114496,
6
+ "after": {
7
+ "active_bytes_current": 42510172672,
8
+ "active_bytes_peak": 127977986048,
9
+ "allocated_bytes_current": 42510172672,
10
+ "allocated_bytes_peak": 124842047488,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 18418384384,
13
+ "inactive_split_bytes_peak": 42846909952,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 150976069632,
21
+ "reserved_bytes_peak": 150976069632,
22
+ "segment_count_current": 1113,
23
+ "segment_count_peak": 1113,
24
+ "timestamp_utc": "2026-08-26T19:01:26.509376Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42510172672,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 124842047488,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42492211712,
33
+ "active_bytes_peak": 125118871552,
34
+ "allocated_bytes_current": 42492211712,
35
+ "allocated_bytes_peak": 124842047488,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 16358067712,
38
+ "inactive_split_bytes_peak": 28274912256,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 129805320192,
46
+ "reserved_bytes_peak": 129805320192,
47
+ "segment_count_current": 930,
48
+ "segment_count_peak": 930,
49
+ "timestamp_utc": "2026-08-26T19:01:24.568936Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18418384384,
55
+ "inactive_split_bytes_current_delta": 2060316672,
56
+ "inactive_split_bytes_peak": 42846909952,
57
+ "inactive_split_bytes_peak_delta": 14571997696,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 150976069632,
68
+ "reserved_bytes_current_delta": 21170749440,
69
+ "reserved_bytes_peak": 150976069632,
70
+ "reserved_bytes_peak_delta": 21170749440,
71
+ "segment_count_current": 1113,
72
+ "segment_count_current_delta": 183,
73
+ "segment_count_peak": 1113,
74
+ "segment_count_peak_delta": 183,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:26.594682Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:925adec3bbdc894d9b03b4822e74824f84c6869ad3a9e54dc7e09228e1a1fda8
3
+ size 4733954
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42510172672,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127977986048,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42510172672,
8
+ "active_bytes_peak": 127977986048,
9
+ "allocated_bytes_current": 42510172672,
10
+ "allocated_bytes_peak": 124842047488,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 18418384384,
13
+ "inactive_split_bytes_peak": 42846909952,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 150976069632,
21
+ "reserved_bytes_peak": 150976069632,
22
+ "segment_count_current": 1113,
23
+ "segment_count_peak": 1113,
24
+ "timestamp_utc": "2026-08-26T19:01:27.191276Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42510172672,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124842047488,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42510172672,
33
+ "active_bytes_peak": 127977986048,
34
+ "allocated_bytes_current": 42510172672,
35
+ "allocated_bytes_peak": 124842047488,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 18418384384,
38
+ "inactive_split_bytes_peak": 42846909952,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 150976069632,
46
+ "reserved_bytes_peak": 150976069632,
47
+ "segment_count_current": 1113,
48
+ "segment_count_peak": 1113,
49
+ "timestamp_utc": "2026-08-26T19:01:26.606379Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18418384384,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 42846909952,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 150976069632,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 150976069632,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1113,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1113,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.201475Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a66ee1dcece15c4ed708c15314ed0b969cadee97d4e9036ab8a85455cd2a6a45
3
+ size 376846
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42510172672,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127977986048,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42510172672,
8
+ "active_bytes_peak": 127977986048,
9
+ "allocated_bytes_current": 42510172672,
10
+ "allocated_bytes_peak": 124842047488,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 18418384384,
13
+ "inactive_split_bytes_peak": 42846909952,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 150976069632,
21
+ "reserved_bytes_peak": 150976069632,
22
+ "segment_count_current": 1113,
23
+ "segment_count_peak": 1113,
24
+ "timestamp_utc": "2026-08-26T19:01:27.792498Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42510172672,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 124842047488,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42510172672,
33
+ "active_bytes_peak": 127977986048,
34
+ "allocated_bytes_current": 42510172672,
35
+ "allocated_bytes_peak": 124842047488,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 18418384384,
38
+ "inactive_split_bytes_peak": 42846909952,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 150976069632,
46
+ "reserved_bytes_peak": 150976069632,
47
+ "segment_count_current": 1113,
48
+ "segment_count_peak": 1113,
49
+ "timestamp_utc": "2026-08-26T19:01:27.202580Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18418384384,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 42846909952,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 150976069632,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 150976069632,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1113,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1113,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:27.802750Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d71623ea70e2bd1904739534d921c7c091502d45bd9de68e24869bddb6274cc7
3
+ size 376898
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42496943104,
3
+ "active_bytes_current_delta": 19448559616,
4
+ "active_bytes_peak": 110649736192,
5
+ "active_bytes_peak_delta": 87588769792,
6
+ "after": {
7
+ "active_bytes_current": 42496943104,
8
+ "active_bytes_peak": 110649736192,
9
+ "allocated_bytes_current": 42496943104,
10
+ "allocated_bytes_peak": 110649736192,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 16261061632,
13
+ "inactive_split_bytes_peak": 28203455488,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 115263668224,
21
+ "reserved_bytes_peak": 115263668224,
22
+ "segment_count_current": 919,
23
+ "segment_count_peak": 919,
24
+ "timestamp_utc": "2026-08-26T19:01:23.031504Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42496943104,
28
+ "allocated_bytes_current_delta": 19448559616,
29
+ "allocated_bytes_peak": 110649736192,
30
+ "allocated_bytes_peak_delta": 87588769792,
31
+ "before": {
32
+ "active_bytes_current": 23048383488,
33
+ "active_bytes_peak": 23060966400,
34
+ "allocated_bytes_current": 23048383488,
35
+ "allocated_bytes_peak": 23060966400,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 376804352,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 25623003136,
46
+ "reserved_bytes_peak": 25623003136,
47
+ "segment_count_current": 551,
48
+ "segment_count_peak": 551,
49
+ "timestamp_utc": "2026-08-26T19:00:42.978679Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16261061632,
55
+ "inactive_split_bytes_current_delta": 15884257280,
56
+ "inactive_split_bytes_peak": 28203455488,
57
+ "inactive_split_bytes_peak_delta": 27320220160,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115263668224,
68
+ "reserved_bytes_current_delta": 89640665088,
69
+ "reserved_bytes_peak": 115263668224,
70
+ "reserved_bytes_peak_delta": 89640665088,
71
+ "segment_count_current": 919,
72
+ "segment_count_current_delta": 368,
73
+ "segment_count_peak": 919,
74
+ "segment_count_peak_delta": 368,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:01:23.114526Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }