AbhishekGhosh1998 commited on
Commit
cd778d8
·
verified ·
1 Parent(s): 25086c2

granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +152 -0
  2. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.json +79 -0
  3. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle +3 -0
  4. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.json +79 -0
  5. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle +3 -0
  6. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.json +79 -0
  7. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle +3 -0
  8. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.json +79 -0
  9. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle +3 -0
  10. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.json +79 -0
  11. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle +3 -0
  12. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.json +79 -0
  13. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle +3 -0
  14. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.json +79 -0
  15. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle +3 -0
  16. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.json +79 -0
  17. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle +3 -0
  18. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.json +79 -0
  19. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle +3 -0
  20. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.json +79 -0
  21. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle +3 -0
  22. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.json +79 -0
  23. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle +3 -0
  24. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.json +79 -0
  25. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle +3 -0
  26. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.json +79 -0
  27. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle +3 -0
  28. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.json +79 -0
  29. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle +3 -0
  30. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.json +79 -0
  31. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle +3 -0
  32. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.json +79 -0
  33. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle +3 -0
  34. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.json +79 -0
  35. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle +3 -0
  36. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.json +79 -0
  37. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle +3 -0
  38. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.json +79 -0
  39. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle +3 -0
  40. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.json +79 -0
  41. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle +3 -0
  42. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.json +79 -0
  43. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle +3 -0
  44. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.json +79 -0
  45. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle +3 -0
  46. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.json +79 -0
  47. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle +3 -0
  48. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.json +79 -0
  49. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle +3 -0
  50. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.json +79 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv CHANGED
@@ -94,3 +94,155 @@
94
  5, 54107
95
  6, 54107
96
  7, 54107
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
  5, 54107
95
  6, 54107
96
  7, 54107
97
+ 0, 54479
98
+ 1, 54497
99
+ 2, 54469
100
+ 3, 54483
101
+ 4, 54191
102
+ 5, 54499
103
+ 6, 54473
104
+ 7, 54487
105
+ 0, 55545
106
+ 1, 55545
107
+ 2, 55545
108
+ 3, 55545
109
+ 4, 55257
110
+ 5, 55545
111
+ 6, 55545
112
+ 7, 55545
113
+ 0, 56057
114
+ 1, 56057
115
+ 2, 56057
116
+ 3, 56057
117
+ 4, 55769
118
+ 5, 56057
119
+ 6, 56057
120
+ 7, 56057
121
+ 0, 56971
122
+ 1, 56971
123
+ 2, 56971
124
+ 3, 56971
125
+ 4, 56683
126
+ 5, 56971
127
+ 6, 56971
128
+ 7, 56971
129
+ 0, 57593
130
+ 1, 57591
131
+ 2, 57591
132
+ 3, 57591
133
+ 4, 57305
134
+ 5, 57591
135
+ 6, 57591
136
+ 7, 57591
137
+ 0, 96129
138
+ 1, 94601
139
+ 2, 94601
140
+ 3, 94601
141
+ 4, 94573
142
+ 5, 94601
143
+ 6, 96173
144
+ 7, 94601
145
+ 0, 96841
146
+ 1, 96885
147
+ 2, 96885
148
+ 3, 96885
149
+ 4, 96857
150
+ 5, 96885
151
+ 6, 96885
152
+ 7, 96885
153
+ 0, 96841
154
+ 1, 96885
155
+ 2, 96885
156
+ 3, 96885
157
+ 4, 96857
158
+ 5, 96885
159
+ 6, 96885
160
+ 7, 96885
161
+ 0, 96841
162
+ 1, 96885
163
+ 2, 96885
164
+ 3, 96885
165
+ 4, 96857
166
+ 5, 96885
167
+ 6, 96885
168
+ 7, 96885
169
+ 0, 96841
170
+ 1, 96885
171
+ 2, 96885
172
+ 3, 96885
173
+ 4, 96857
174
+ 5, 96885
175
+ 6, 96885
176
+ 7, 96885
177
+ 0, 96841
178
+ 1, 96885
179
+ 2, 96885
180
+ 3, 96885
181
+ 4, 96857
182
+ 5, 96885
183
+ 6, 96885
184
+ 7, 96885
185
+ 0, 98191
186
+ 1, 98235
187
+ 2, 98235
188
+ 3, 98235
189
+ 4, 98207
190
+ 5, 98235
191
+ 6, 98235
192
+ 7, 98235
193
+ 0, 101927
194
+ 1, 101971
195
+ 2, 101971
196
+ 3, 101971
197
+ 4, 101943
198
+ 5, 101971
199
+ 6, 101971
200
+ 7, 101971
201
+ 0, 108281
202
+ 1, 108393
203
+ 2, 108349
204
+ 3, 108345
205
+ 4, 108419
206
+ 5, 108345
207
+ 6, 108391
208
+ 7, 108403
209
+ 0, 120723
210
+ 1, 120415
211
+ 2, 120335
212
+ 3, 119923
213
+ 4, 119591
214
+ 5, 120455
215
+ 6, 120755
216
+ 7, 119891
217
+ 0, 145005
218
+ 1, 145199
219
+ 2, 145199
220
+ 3, 145199
221
+ 4, 145347
222
+ 5, 145199
223
+ 6, 145199
224
+ 7, 145199
225
+ 0, 143759
226
+ 1, 143953
227
+ 2, 143953
228
+ 3, 143953
229
+ 4, 144165
230
+ 5, 144599
231
+ 6, 144273
232
+ 7, 143953
233
+ 0, 4643
234
+ 1, 3366
235
+ 2, 3366
236
+ 3, 3366
237
+ 4, 3366
238
+ 5, 3366
239
+ 6, 3622
240
+ 7, 143951
241
+ 0, 0
242
+ 1, 0
243
+ 2, 0
244
+ 3, 0
245
+ 4, 0
246
+ 5, 0
247
+ 6, 0
248
+ 7, 0
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85292954112,
3
+ "active_bytes_current_delta": 40772344320,
4
+ "active_bytes_peak": 90610880000,
5
+ "active_bytes_peak_delta": 46065104384,
6
+ "after": {
7
+ "active_bytes_current": 85292954112,
8
+ "active_bytes_peak": 90610880000,
9
+ "allocated_bytes_current": 85292954112,
10
+ "allocated_bytes_peak": 90610880000,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1365560832,
13
+ "inactive_split_bytes_peak": 11201188352,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 103173586944,
21
+ "reserved_bytes_peak": 103173586944,
22
+ "segment_count_current": 1786,
23
+ "segment_count_peak": 1786,
24
+ "timestamp_utc": "2026-08-26T18:50:34.927898Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85292954112,
28
+ "allocated_bytes_current_delta": 40772344320,
29
+ "allocated_bytes_peak": 90610880000,
30
+ "allocated_bytes_peak_delta": 46065104384,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 446523392,
38
+ "inactive_split_bytes_peak": 1050503168,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T18:49:54.861513Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1365560832,
55
+ "inactive_split_bytes_current_delta": 919037440,
56
+ "inactive_split_bytes_peak": 11201188352,
57
+ "inactive_split_bytes_peak_delta": 10150685184,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103173586944,
68
+ "reserved_bytes_current_delta": 54922313728,
69
+ "reserved_bytes_peak": 103173586944,
70
+ "reserved_bytes_peak_delta": 54922313728,
71
+ "segment_count_current": 1786,
72
+ "segment_count_current_delta": 1165,
73
+ "segment_count_peak": 1786,
74
+ "segment_count_peak_delta": 1165,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:34.981197Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0aa4a4bb58a42d4afb2348619f36704949c1954e13d6810a889605bef2a6594a
3
+ size 3160743
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85292954112,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121477026816,
5
+ "active_bytes_peak_delta": 30866146816,
6
+ "after": {
7
+ "active_bytes_current": 85292954112,
8
+ "active_bytes_peak": 121477026816,
9
+ "allocated_bytes_current": 85292954112,
10
+ "allocated_bytes_peak": 121477026816,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1770311168,
13
+ "inactive_split_bytes_peak": 11201188352,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 128687538176,
21
+ "reserved_bytes_peak": 128687538176,
22
+ "segment_count_current": 2005,
23
+ "segment_count_peak": 2005,
24
+ "timestamp_utc": "2026-08-26T18:50:35.546179Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85292954112,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121477026816,
30
+ "allocated_bytes_peak_delta": 30866146816,
31
+ "before": {
32
+ "active_bytes_current": 85292954112,
33
+ "active_bytes_peak": 90610880000,
34
+ "allocated_bytes_current": 85292954112,
35
+ "allocated_bytes_peak": 90610880000,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1365560832,
38
+ "inactive_split_bytes_peak": 11201188352,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 103173586944,
46
+ "reserved_bytes_peak": 103173586944,
47
+ "segment_count_current": 1786,
48
+ "segment_count_peak": 1786,
49
+ "timestamp_utc": "2026-08-26T18:50:34.987192Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1770311168,
55
+ "inactive_split_bytes_current_delta": 404750336,
56
+ "inactive_split_bytes_peak": 11201188352,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128687538176,
68
+ "reserved_bytes_current_delta": 25513951232,
69
+ "reserved_bytes_peak": 128687538176,
70
+ "reserved_bytes_peak_delta": 25513951232,
71
+ "segment_count_current": 2005,
72
+ "segment_count_current_delta": 219,
73
+ "segment_count_peak": 2005,
74
+ "segment_count_peak_delta": 219,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:35.595543Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c2ab06c8c945ecdb672016c8f54cb38720c97c1653b13de54badb3b299e51a4
3
+ size 2586991
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85292954112,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121477075968,
5
+ "active_bytes_peak_delta": 49152,
6
+ "after": {
7
+ "active_bytes_current": 85292954112,
8
+ "active_bytes_peak": 121477075968,
9
+ "allocated_bytes_current": 85292954112,
10
+ "allocated_bytes_peak": 121477075968,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1770311168,
13
+ "inactive_split_bytes_peak": 11201188352,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 128687538176,
21
+ "reserved_bytes_peak": 128687538176,
22
+ "segment_count_current": 2005,
23
+ "segment_count_peak": 2005,
24
+ "timestamp_utc": "2026-08-26T18:50:35.971739Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85292954112,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121477075968,
30
+ "allocated_bytes_peak_delta": 49152,
31
+ "before": {
32
+ "active_bytes_current": 85292954112,
33
+ "active_bytes_peak": 121477026816,
34
+ "allocated_bytes_current": 85292954112,
35
+ "allocated_bytes_peak": 121477026816,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1770311168,
38
+ "inactive_split_bytes_peak": 11201188352,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 128687538176,
46
+ "reserved_bytes_peak": 128687538176,
47
+ "segment_count_current": 2005,
48
+ "segment_count_peak": 2005,
49
+ "timestamp_utc": "2026-08-26T18:50:35.601397Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1770311168,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11201188352,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128687538176,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 128687538176,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2005,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2005,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:36.017227Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e91cae5e5b40799ac6bd86fc41b310aa28ee2e4b11be02b9bd4055f177b4cb5
3
+ size 2609276
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85309997568,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 128733144064,
5
+ "active_bytes_peak_delta": 7256068096,
6
+ "after": {
7
+ "active_bytes_current": 85309997568,
8
+ "active_bytes_peak": 128733144064,
9
+ "allocated_bytes_current": 85309997568,
10
+ "allocated_bytes_peak": 121485386752,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1453374976,
13
+ "inactive_split_bytes_peak": 12882581504,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 141482262528,
21
+ "reserved_bytes_peak": 141482262528,
22
+ "segment_count_current": 1805,
23
+ "segment_count_peak": 2005,
24
+ "timestamp_utc": "2026-08-26T18:50:38.956995Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85309997568,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 121485386752,
30
+ "allocated_bytes_peak_delta": 8310784,
31
+ "before": {
32
+ "active_bytes_current": 85292954112,
33
+ "active_bytes_peak": 121477075968,
34
+ "allocated_bytes_current": 85292954112,
35
+ "allocated_bytes_peak": 121477075968,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1770311168,
38
+ "inactive_split_bytes_peak": 11201188352,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 128687538176,
46
+ "reserved_bytes_peak": 128687538176,
47
+ "segment_count_current": 2005,
48
+ "segment_count_peak": 2005,
49
+ "timestamp_utc": "2026-08-26T18:50:36.022500Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1453374976,
55
+ "inactive_split_bytes_current_delta": -316936192,
56
+ "inactive_split_bytes_peak": 12882581504,
57
+ "inactive_split_bytes_peak_delta": 1681393152,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141482262528,
68
+ "reserved_bytes_current_delta": 12794724352,
69
+ "reserved_bytes_peak": 141482262528,
70
+ "reserved_bytes_peak_delta": 12794724352,
71
+ "segment_count_current": 1805,
72
+ "segment_count_current_delta": -200,
73
+ "segment_count_peak": 2005,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.010451Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e3a5548b896b2a25ca4a2fb6b2899d86e34f93bf80fffcbbb54656f939a519c
3
+ size 2779026
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85309997568,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128733144064,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85309997568,
8
+ "active_bytes_peak": 128733144064,
9
+ "allocated_bytes_current": 85309997568,
10
+ "allocated_bytes_peak": 121485386752,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1453374976,
13
+ "inactive_split_bytes_peak": 12882581504,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 141482262528,
21
+ "reserved_bytes_peak": 141482262528,
22
+ "segment_count_current": 1805,
23
+ "segment_count_peak": 2005,
24
+ "timestamp_utc": "2026-08-26T18:50:39.331583Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85309997568,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121485386752,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85309997568,
33
+ "active_bytes_peak": 128733144064,
34
+ "allocated_bytes_current": 85309997568,
35
+ "allocated_bytes_peak": 121485386752,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1453374976,
38
+ "inactive_split_bytes_peak": 12882581504,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 141482262528,
46
+ "reserved_bytes_peak": 141482262528,
47
+ "segment_count_current": 1805,
48
+ "segment_count_peak": 2005,
49
+ "timestamp_utc": "2026-08-26T18:50:39.017489Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1453374976,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12882581504,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141482262528,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141482262528,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1805,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2005,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.342045Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6183e257ea248a054b02716706577af577188c192e256577c3ffa6d94ccb4de7
3
+ size 422083
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85309997568,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128733144064,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85309997568,
8
+ "active_bytes_peak": 128733144064,
9
+ "allocated_bytes_current": 85309997568,
10
+ "allocated_bytes_peak": 121485386752,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1453374976,
13
+ "inactive_split_bytes_peak": 12882581504,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 141482262528,
21
+ "reserved_bytes_peak": 141482262528,
22
+ "segment_count_current": 1805,
23
+ "segment_count_peak": 2005,
24
+ "timestamp_utc": "2026-08-26T18:50:39.652667Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85309997568,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121485386752,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85309997568,
33
+ "active_bytes_peak": 128733144064,
34
+ "allocated_bytes_current": 85309997568,
35
+ "allocated_bytes_peak": 121485386752,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1453374976,
38
+ "inactive_split_bytes_peak": 12882581504,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 141482262528,
46
+ "reserved_bytes_peak": 141482262528,
47
+ "segment_count_current": 1805,
48
+ "segment_count_peak": 2005,
49
+ "timestamp_utc": "2026-08-26T18:50:39.343193Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1453374976,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12882581504,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141482262528,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141482262528,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1805,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2005,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.662359Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bd61744ce6d87e4c745b1d183902c4d00ccc1bd8a56aad6e8a91430ad96fbb6
3
+ size 422083
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 40774371840,
4
+ "active_bytes_peak": 90612776448,
5
+ "active_bytes_peak_delta": 46067000832,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 90612776448,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 90612776448,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1566957056,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 103244890112,
21
+ "reserved_bytes_peak": 103244890112,
22
+ "segment_count_current": 1790,
23
+ "segment_count_peak": 1790,
24
+ "timestamp_utc": "2026-08-26T18:50:34.927803Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 40774371840,
29
+ "allocated_bytes_peak": 90612776448,
30
+ "allocated_bytes_peak_delta": 46067000832,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 421357568,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T18:49:54.861521Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1566957056,
55
+ "inactive_split_bytes_current_delta": 1145599488,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 10888034816,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103244890112,
68
+ "reserved_bytes_current_delta": 54993616896,
69
+ "reserved_bytes_peak": 103244890112,
70
+ "reserved_bytes_peak_delta": 54993616896,
71
+ "segment_count_current": 1790,
72
+ "segment_count_current_delta": 1169,
73
+ "segment_count_peak": 1790,
74
+ "segment_count_peak_delta": 1169,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:34.980104Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd97ae4b71d75d8df2690b1b9bd0903c3090a75cbc027e10b301f7c4dccd2865
3
+ size 3148002
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121478210048,
5
+ "active_bytes_peak_delta": 30865433600,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 121478210048,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 121478210048,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1969610240,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 128804978688,
21
+ "reserved_bytes_peak": 128804978688,
22
+ "segment_count_current": 2012,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:35.546159Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121478210048,
30
+ "allocated_bytes_peak_delta": 30865433600,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 90612776448,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 90612776448,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1566957056,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 103244890112,
46
+ "reserved_bytes_peak": 103244890112,
47
+ "segment_count_current": 1790,
48
+ "segment_count_peak": 1790,
49
+ "timestamp_utc": "2026-08-26T18:50:34.986365Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1969610240,
55
+ "inactive_split_bytes_current_delta": 402653184,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128804978688,
68
+ "reserved_bytes_current_delta": 25560088576,
69
+ "reserved_bytes_peak": 128804978688,
70
+ "reserved_bytes_peak_delta": 25560088576,
71
+ "segment_count_current": 2012,
72
+ "segment_count_current_delta": 222,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 222,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:35.596037Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af77a838b6c6a61374c9704ed59f260069beb78734595ba0949d7d8f173b1c0b
3
+ size 2579979
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121478455808,
5
+ "active_bytes_peak_delta": 245760,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 121478455808,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 121478455808,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1971707392,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 128804978688,
21
+ "reserved_bytes_peak": 128804978688,
22
+ "segment_count_current": 2012,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:35.971660Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121478455808,
30
+ "allocated_bytes_peak_delta": 245760,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 121478210048,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 121478210048,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1969610240,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 128804978688,
46
+ "reserved_bytes_peak": 128804978688,
47
+ "segment_count_current": 2012,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:35.602405Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1971707392,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128804978688,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 128804978688,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2012,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:36.017792Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8705912aae623c560ae0d7d576d6e67e34ff869f0e694974c17f0d2cccc07fd0
3
+ size 2569019
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 7256715776,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 13083977728,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:38.956984Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 8958464,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 121478455808,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 121478455808,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1971707392,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 128804978688,
46
+ "reserved_bytes_peak": 128804978688,
47
+ "segment_count_current": 2012,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:36.022865Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": -319033344,
56
+ "inactive_split_bytes_peak": 13083977728,
57
+ "inactive_split_bytes_peak_delta": 1170605568,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 12880707584,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 12880707584,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": -197,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.010026Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29bc1811482194f01ba57189d5b0336f0a556b13bcd06f639573a807a68f2d02
3
+ size 2771672
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 13083977728,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:39.331461Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312025088,
33
+ "active_bytes_peak": 128735171584,
34
+ "allocated_bytes_current": 85312025088,
35
+ "allocated_bytes_peak": 121487414272,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1652674048,
38
+ "inactive_split_bytes_peak": 13083977728,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 141685686272,
46
+ "reserved_bytes_peak": 141685686272,
47
+ "segment_count_current": 1815,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:39.015933Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13083977728,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.341247Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f31fa3e47c17b389551300bd90296f5b7d4a52bd842eebe30fe68356467f07a0
3
+ size 416145
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 13083977728,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:39.652649Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312025088,
33
+ "active_bytes_peak": 128735171584,
34
+ "allocated_bytes_current": 85312025088,
35
+ "allocated_bytes_peak": 121487414272,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1652674048,
38
+ "inactive_split_bytes_peak": 13083977728,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 141685686272,
46
+ "reserved_bytes_peak": 141685686272,
47
+ "segment_count_current": 1815,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:39.342351Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13083977728,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.662081Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52f33d0f617a93d8d768f32da49c5a936943607e881cc1a361ba3301d7d47d13
3
+ size 416145
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 40774371840,
4
+ "active_bytes_peak": 90612776448,
5
+ "active_bytes_peak_delta": 46067000832,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 90612776448,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 90612776448,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1566957056,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 103244890112,
21
+ "reserved_bytes_peak": 103244890112,
22
+ "segment_count_current": 1790,
23
+ "segment_count_peak": 1790,
24
+ "timestamp_utc": "2026-08-26T18:50:34.927806Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 40774371840,
29
+ "allocated_bytes_peak": 90612776448,
30
+ "allocated_bytes_peak_delta": 46067000832,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 421357568,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T18:49:54.861483Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1566957056,
55
+ "inactive_split_bytes_current_delta": 1145599488,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 10888034816,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103244890112,
68
+ "reserved_bytes_current_delta": 54993616896,
69
+ "reserved_bytes_peak": 103244890112,
70
+ "reserved_bytes_peak_delta": 54993616896,
71
+ "segment_count_current": 1790,
72
+ "segment_count_current_delta": 1169,
73
+ "segment_count_peak": 1790,
74
+ "segment_count_peak_delta": 1169,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:34.981556Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27b146b2dfaeba3efc68604c5b3c53fb4d0bbb10f8c3a1b64518f0b3c3060ef1
3
+ size 3148002
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121478210048,
5
+ "active_bytes_peak_delta": 30865433600,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 121478210048,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 121478210048,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1969610240,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 128804978688,
21
+ "reserved_bytes_peak": 128804978688,
22
+ "segment_count_current": 2012,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:35.546227Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121478210048,
30
+ "allocated_bytes_peak_delta": 30865433600,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 90612776448,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 90612776448,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1566957056,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 103244890112,
46
+ "reserved_bytes_peak": 103244890112,
47
+ "segment_count_current": 1790,
48
+ "segment_count_peak": 1790,
49
+ "timestamp_utc": "2026-08-26T18:50:34.987886Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1969610240,
55
+ "inactive_split_bytes_current_delta": 402653184,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128804978688,
68
+ "reserved_bytes_current_delta": 25560088576,
69
+ "reserved_bytes_peak": 128804978688,
70
+ "reserved_bytes_peak_delta": 25560088576,
71
+ "segment_count_current": 2012,
72
+ "segment_count_current_delta": 222,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 222,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:35.595565Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5b1e78a390fe971af9f4fc30d95e219aac098eba527b9f930025a3321f93456
3
+ size 2579948
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121478455808,
5
+ "active_bytes_peak_delta": 245760,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 121478455808,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 121478455808,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1971707392,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 128804978688,
21
+ "reserved_bytes_peak": 128804978688,
22
+ "segment_count_current": 2012,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:35.971658Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121478455808,
30
+ "allocated_bytes_peak_delta": 245760,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 121478210048,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 121478210048,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1969610240,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 128804978688,
46
+ "reserved_bytes_peak": 128804978688,
47
+ "segment_count_current": 2012,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:35.601963Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1971707392,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128804978688,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 128804978688,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2012,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:36.016491Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:249e9b8f6d474565c05df97910be1857323ef733f2bf81f86f2e60db2e0c0a4f
3
+ size 2569055
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 7256715776,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 13083977728,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:38.956987Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 8958464,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 121478455808,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 121478455808,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1971707392,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 128804978688,
46
+ "reserved_bytes_peak": 128804978688,
47
+ "segment_count_current": 2012,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:36.021547Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": -319033344,
56
+ "inactive_split_bytes_peak": 13083977728,
57
+ "inactive_split_bytes_peak_delta": 1170605568,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 12880707584,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 12880707584,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": -197,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.010224Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4861d2922ba964c05d12e06573ce96afcd744654d2763ee57e9b0332af50b0e9
3
+ size 2771672
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 13083977728,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:39.331456Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312025088,
33
+ "active_bytes_peak": 128735171584,
34
+ "allocated_bytes_current": 85312025088,
35
+ "allocated_bytes_peak": 121487414272,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1652674048,
38
+ "inactive_split_bytes_peak": 13083977728,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 141685686272,
46
+ "reserved_bytes_peak": 141685686272,
47
+ "segment_count_current": 1815,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:39.016318Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13083977728,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.341347Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:91b5d7b1733ad80c6a16d831fe2407c6b08974fd6fc2484f9afd62e0a6f0c9d0
3
+ size 416145
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 13083977728,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:39.652654Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312025088,
33
+ "active_bytes_peak": 128735171584,
34
+ "allocated_bytes_current": 85312025088,
35
+ "allocated_bytes_peak": 121487414272,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1652674048,
38
+ "inactive_split_bytes_peak": 13083977728,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 141685686272,
46
+ "reserved_bytes_peak": 141685686272,
47
+ "segment_count_current": 1815,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:39.342440Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13083977728,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.662013Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48bea32c141f6c0dc478293871003f1d61d30ebf5a6daf2e6748d169924c2102
3
+ size 416145
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 40774371840,
4
+ "active_bytes_peak": 90612776448,
5
+ "active_bytes_peak_delta": 46067000832,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 90612776448,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 90612776448,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1566957056,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 103244890112,
21
+ "reserved_bytes_peak": 103244890112,
22
+ "segment_count_current": 1790,
23
+ "segment_count_peak": 1790,
24
+ "timestamp_utc": "2026-08-26T18:50:34.927781Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 40774371840,
29
+ "allocated_bytes_peak": 90612776448,
30
+ "allocated_bytes_peak_delta": 46067000832,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 421357568,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T18:49:54.861469Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1566957056,
55
+ "inactive_split_bytes_current_delta": 1145599488,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 10888034816,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103244890112,
68
+ "reserved_bytes_current_delta": 54993616896,
69
+ "reserved_bytes_peak": 103244890112,
70
+ "reserved_bytes_peak_delta": 54993616896,
71
+ "segment_count_current": 1790,
72
+ "segment_count_current_delta": 1169,
73
+ "segment_count_peak": 1790,
74
+ "segment_count_peak_delta": 1169,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:34.979803Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bd36093f315ca9b352532172956de6dfd4183b9176483c7a47e15c64fba50af
3
+ size 3147999
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121478210048,
5
+ "active_bytes_peak_delta": 30865433600,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 121478210048,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 121478210048,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1969610240,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 128804978688,
21
+ "reserved_bytes_peak": 128804978688,
22
+ "segment_count_current": 2012,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:35.546147Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121478210048,
30
+ "allocated_bytes_peak_delta": 30865433600,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 90612776448,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 90612776448,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1566957056,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 103244890112,
46
+ "reserved_bytes_peak": 103244890112,
47
+ "segment_count_current": 1790,
48
+ "segment_count_peak": 1790,
49
+ "timestamp_utc": "2026-08-26T18:50:34.986030Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1969610240,
55
+ "inactive_split_bytes_current_delta": 402653184,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128804978688,
68
+ "reserved_bytes_current_delta": 25560088576,
69
+ "reserved_bytes_peak": 128804978688,
70
+ "reserved_bytes_peak_delta": 25560088576,
71
+ "segment_count_current": 2012,
72
+ "segment_count_current_delta": 222,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 222,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:35.594867Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e587809f83fdda618266730bd28e2282bd753e40bc02169039a8e4b34ceaf25d
3
+ size 2579979
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85294981632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122082435584,
5
+ "active_bytes_peak_delta": 604225536,
6
+ "after": {
7
+ "active_bytes_current": 85294981632,
8
+ "active_bytes_peak": 122082435584,
9
+ "allocated_bytes_current": 85294981632,
10
+ "allocated_bytes_peak": 121478455808,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1971707392,
13
+ "inactive_split_bytes_peak": 11913372160,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 128804978688,
21
+ "reserved_bytes_peak": 128804978688,
22
+ "segment_count_current": 2012,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:35.971654Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85294981632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121478455808,
30
+ "allocated_bytes_peak_delta": 245760,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 121478210048,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 121478210048,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1969610240,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 128804978688,
46
+ "reserved_bytes_peak": 128804978688,
47
+ "segment_count_current": 2012,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:35.601253Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1971707392,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 11913372160,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 128804978688,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 128804978688,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2012,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:36.016411Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ebc496a7c1c0b713c075ec11ebed6a4aca0be3a6c15bf3abbe78aafc67ca8b8
3
+ size 2569019
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 6652736000,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 12815542272,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:38.956989Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 8958464,
31
+ "before": {
32
+ "active_bytes_current": 85294981632,
33
+ "active_bytes_peak": 122082435584,
34
+ "allocated_bytes_current": 85294981632,
35
+ "allocated_bytes_peak": 121478455808,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1971707392,
38
+ "inactive_split_bytes_peak": 11913372160,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 128804978688,
46
+ "reserved_bytes_peak": 128804978688,
47
+ "segment_count_current": 2012,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:36.021568Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": -319033344,
56
+ "inactive_split_bytes_peak": 12815542272,
57
+ "inactive_split_bytes_peak_delta": 902170112,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 12880707584,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 12880707584,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": -197,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.010626Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e19767dc90ece8d439f6886a341dd416885067ba2615502ae69deab56959ca22
3
+ size 2771669
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 12815542272,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:39.331469Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312025088,
33
+ "active_bytes_peak": 128735171584,
34
+ "allocated_bytes_current": 85312025088,
35
+ "allocated_bytes_peak": 121487414272,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1652674048,
38
+ "inactive_split_bytes_peak": 12815542272,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 141685686272,
46
+ "reserved_bytes_peak": 141685686272,
47
+ "segment_count_current": 1815,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:39.016682Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12815542272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.341608Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ba49380f0f1c7c224d6180fa854e53bfa52c31ea95d95b166dfceece6a8f2e2
3
+ size 416035
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312025088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 128735171584,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312025088,
8
+ "active_bytes_peak": 128735171584,
9
+ "allocated_bytes_current": 85312025088,
10
+ "allocated_bytes_peak": 121487414272,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1652674048,
13
+ "inactive_split_bytes_peak": 12815542272,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 141685686272,
21
+ "reserved_bytes_peak": 141685686272,
22
+ "segment_count_current": 1815,
23
+ "segment_count_peak": 2012,
24
+ "timestamp_utc": "2026-08-26T18:50:39.652658Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312025088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121487414272,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312025088,
33
+ "active_bytes_peak": 128735171584,
34
+ "allocated_bytes_current": 85312025088,
35
+ "allocated_bytes_peak": 121487414272,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1652674048,
38
+ "inactive_split_bytes_peak": 12815542272,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 141685686272,
46
+ "reserved_bytes_peak": 141685686272,
47
+ "segment_count_current": 1815,
48
+ "segment_count_peak": 2012,
49
+ "timestamp_utc": "2026-08-26T18:50:39.342717Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1652674048,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12815542272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 141685686272,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 141685686272,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1815,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2012,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:39.662417Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77688a743792e2b71ed5652b03f4fb807657b5a012cf454a3c1eceb47d63c720
3
+ size 416035
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295019520,
3
+ "active_bytes_current_delta": 40774409728,
4
+ "active_bytes_peak": 90623401472,
5
+ "active_bytes_peak_delta": 46077625856,
6
+ "after": {
7
+ "active_bytes_current": 85295019520,
8
+ "active_bytes_peak": 90623401472,
9
+ "allocated_bytes_current": 85295019520,
10
+ "allocated_bytes_peak": 90623401472,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1720011264,
13
+ "inactive_split_bytes_peak": 11677395968,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 103291027456,
21
+ "reserved_bytes_peak": 103291027456,
22
+ "segment_count_current": 1798,
23
+ "segment_count_peak": 1798,
24
+ "timestamp_utc": "2026-08-26T18:50:34.927754Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295019520,
28
+ "allocated_bytes_current_delta": 40774409728,
29
+ "allocated_bytes_peak": 90623401472,
30
+ "allocated_bytes_peak_delta": 46077625856,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 270362624,
38
+ "inactive_split_bytes_peak": 874342400,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 47949283328,
46
+ "reserved_bytes_peak": 47949283328,
47
+ "segment_count_current": 615,
48
+ "segment_count_peak": 615,
49
+ "timestamp_utc": "2026-08-26T18:49:54.861438Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1720011264,
55
+ "inactive_split_bytes_current_delta": 1449648640,
56
+ "inactive_split_bytes_peak": 11677395968,
57
+ "inactive_split_bytes_peak_delta": 10803053568,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103291027456,
68
+ "reserved_bytes_current_delta": 55341744128,
69
+ "reserved_bytes_peak": 103291027456,
70
+ "reserved_bytes_peak_delta": 55341744128,
71
+ "segment_count_current": 1798,
72
+ "segment_count_current_delta": 1183,
73
+ "segment_count_peak": 1798,
74
+ "segment_count_peak_delta": 1183,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:50:34.981424Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }