granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +152 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.json +79 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv
CHANGED
|
@@ -94,3 +94,155 @@
|
|
| 94 |
5, 54107
|
| 95 |
6, 54107
|
| 96 |
7, 54107
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
5, 54107
|
| 95 |
6, 54107
|
| 96 |
7, 54107
|
| 97 |
+
0, 54479
|
| 98 |
+
1, 54497
|
| 99 |
+
2, 54469
|
| 100 |
+
3, 54483
|
| 101 |
+
4, 54191
|
| 102 |
+
5, 54499
|
| 103 |
+
6, 54473
|
| 104 |
+
7, 54487
|
| 105 |
+
0, 55545
|
| 106 |
+
1, 55545
|
| 107 |
+
2, 55545
|
| 108 |
+
3, 55545
|
| 109 |
+
4, 55257
|
| 110 |
+
5, 55545
|
| 111 |
+
6, 55545
|
| 112 |
+
7, 55545
|
| 113 |
+
0, 56057
|
| 114 |
+
1, 56057
|
| 115 |
+
2, 56057
|
| 116 |
+
3, 56057
|
| 117 |
+
4, 55769
|
| 118 |
+
5, 56057
|
| 119 |
+
6, 56057
|
| 120 |
+
7, 56057
|
| 121 |
+
0, 56971
|
| 122 |
+
1, 56971
|
| 123 |
+
2, 56971
|
| 124 |
+
3, 56971
|
| 125 |
+
4, 56683
|
| 126 |
+
5, 56971
|
| 127 |
+
6, 56971
|
| 128 |
+
7, 56971
|
| 129 |
+
0, 57593
|
| 130 |
+
1, 57591
|
| 131 |
+
2, 57591
|
| 132 |
+
3, 57591
|
| 133 |
+
4, 57305
|
| 134 |
+
5, 57591
|
| 135 |
+
6, 57591
|
| 136 |
+
7, 57591
|
| 137 |
+
0, 96129
|
| 138 |
+
1, 94601
|
| 139 |
+
2, 94601
|
| 140 |
+
3, 94601
|
| 141 |
+
4, 94573
|
| 142 |
+
5, 94601
|
| 143 |
+
6, 96173
|
| 144 |
+
7, 94601
|
| 145 |
+
0, 96841
|
| 146 |
+
1, 96885
|
| 147 |
+
2, 96885
|
| 148 |
+
3, 96885
|
| 149 |
+
4, 96857
|
| 150 |
+
5, 96885
|
| 151 |
+
6, 96885
|
| 152 |
+
7, 96885
|
| 153 |
+
0, 96841
|
| 154 |
+
1, 96885
|
| 155 |
+
2, 96885
|
| 156 |
+
3, 96885
|
| 157 |
+
4, 96857
|
| 158 |
+
5, 96885
|
| 159 |
+
6, 96885
|
| 160 |
+
7, 96885
|
| 161 |
+
0, 96841
|
| 162 |
+
1, 96885
|
| 163 |
+
2, 96885
|
| 164 |
+
3, 96885
|
| 165 |
+
4, 96857
|
| 166 |
+
5, 96885
|
| 167 |
+
6, 96885
|
| 168 |
+
7, 96885
|
| 169 |
+
0, 96841
|
| 170 |
+
1, 96885
|
| 171 |
+
2, 96885
|
| 172 |
+
3, 96885
|
| 173 |
+
4, 96857
|
| 174 |
+
5, 96885
|
| 175 |
+
6, 96885
|
| 176 |
+
7, 96885
|
| 177 |
+
0, 96841
|
| 178 |
+
1, 96885
|
| 179 |
+
2, 96885
|
| 180 |
+
3, 96885
|
| 181 |
+
4, 96857
|
| 182 |
+
5, 96885
|
| 183 |
+
6, 96885
|
| 184 |
+
7, 96885
|
| 185 |
+
0, 98191
|
| 186 |
+
1, 98235
|
| 187 |
+
2, 98235
|
| 188 |
+
3, 98235
|
| 189 |
+
4, 98207
|
| 190 |
+
5, 98235
|
| 191 |
+
6, 98235
|
| 192 |
+
7, 98235
|
| 193 |
+
0, 101927
|
| 194 |
+
1, 101971
|
| 195 |
+
2, 101971
|
| 196 |
+
3, 101971
|
| 197 |
+
4, 101943
|
| 198 |
+
5, 101971
|
| 199 |
+
6, 101971
|
| 200 |
+
7, 101971
|
| 201 |
+
0, 108281
|
| 202 |
+
1, 108393
|
| 203 |
+
2, 108349
|
| 204 |
+
3, 108345
|
| 205 |
+
4, 108419
|
| 206 |
+
5, 108345
|
| 207 |
+
6, 108391
|
| 208 |
+
7, 108403
|
| 209 |
+
0, 120723
|
| 210 |
+
1, 120415
|
| 211 |
+
2, 120335
|
| 212 |
+
3, 119923
|
| 213 |
+
4, 119591
|
| 214 |
+
5, 120455
|
| 215 |
+
6, 120755
|
| 216 |
+
7, 119891
|
| 217 |
+
0, 145005
|
| 218 |
+
1, 145199
|
| 219 |
+
2, 145199
|
| 220 |
+
3, 145199
|
| 221 |
+
4, 145347
|
| 222 |
+
5, 145199
|
| 223 |
+
6, 145199
|
| 224 |
+
7, 145199
|
| 225 |
+
0, 143759
|
| 226 |
+
1, 143953
|
| 227 |
+
2, 143953
|
| 228 |
+
3, 143953
|
| 229 |
+
4, 144165
|
| 230 |
+
5, 144599
|
| 231 |
+
6, 144273
|
| 232 |
+
7, 143953
|
| 233 |
+
0, 4643
|
| 234 |
+
1, 3366
|
| 235 |
+
2, 3366
|
| 236 |
+
3, 3366
|
| 237 |
+
4, 3366
|
| 238 |
+
5, 3366
|
| 239 |
+
6, 3622
|
| 240 |
+
7, 143951
|
| 241 |
+
0, 0
|
| 242 |
+
1, 0
|
| 243 |
+
2, 0
|
| 244 |
+
3, 0
|
| 245 |
+
4, 0
|
| 246 |
+
5, 0
|
| 247 |
+
6, 0
|
| 248 |
+
7, 0
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85292954112,
|
| 3 |
+
"active_bytes_current_delta": 40772344320,
|
| 4 |
+
"active_bytes_peak": 90610880000,
|
| 5 |
+
"active_bytes_peak_delta": 46065104384,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85292954112,
|
| 8 |
+
"active_bytes_peak": 90610880000,
|
| 9 |
+
"allocated_bytes_current": 85292954112,
|
| 10 |
+
"allocated_bytes_peak": 90610880000,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1365560832,
|
| 13 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 103173586944,
|
| 21 |
+
"reserved_bytes_peak": 103173586944,
|
| 22 |
+
"segment_count_current": 1786,
|
| 23 |
+
"segment_count_peak": 1786,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:34.927898Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85292954112,
|
| 28 |
+
"allocated_bytes_current_delta": 40772344320,
|
| 29 |
+
"allocated_bytes_peak": 90610880000,
|
| 30 |
+
"allocated_bytes_peak_delta": 46065104384,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 446523392,
|
| 38 |
+
"inactive_split_bytes_peak": 1050503168,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:49:54.861513Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1365560832,
|
| 55 |
+
"inactive_split_bytes_current_delta": 919037440,
|
| 56 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10150685184,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103173586944,
|
| 68 |
+
"reserved_bytes_current_delta": 54922313728,
|
| 69 |
+
"reserved_bytes_peak": 103173586944,
|
| 70 |
+
"reserved_bytes_peak_delta": 54922313728,
|
| 71 |
+
"segment_count_current": 1786,
|
| 72 |
+
"segment_count_current_delta": 1165,
|
| 73 |
+
"segment_count_peak": 1786,
|
| 74 |
+
"segment_count_peak_delta": 1165,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:34.981197Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0aa4a4bb58a42d4afb2348619f36704949c1954e13d6810a889605bef2a6594a
|
| 3 |
+
size 3160743
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85292954112,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121477026816,
|
| 5 |
+
"active_bytes_peak_delta": 30866146816,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85292954112,
|
| 8 |
+
"active_bytes_peak": 121477026816,
|
| 9 |
+
"allocated_bytes_current": 85292954112,
|
| 10 |
+
"allocated_bytes_peak": 121477026816,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1770311168,
|
| 13 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 128687538176,
|
| 21 |
+
"reserved_bytes_peak": 128687538176,
|
| 22 |
+
"segment_count_current": 2005,
|
| 23 |
+
"segment_count_peak": 2005,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.546179Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85292954112,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121477026816,
|
| 30 |
+
"allocated_bytes_peak_delta": 30866146816,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85292954112,
|
| 33 |
+
"active_bytes_peak": 90610880000,
|
| 34 |
+
"allocated_bytes_current": 85292954112,
|
| 35 |
+
"allocated_bytes_peak": 90610880000,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1365560832,
|
| 38 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 103173586944,
|
| 46 |
+
"reserved_bytes_peak": 103173586944,
|
| 47 |
+
"segment_count_current": 1786,
|
| 48 |
+
"segment_count_peak": 1786,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:34.987192Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1770311168,
|
| 55 |
+
"inactive_split_bytes_current_delta": 404750336,
|
| 56 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128687538176,
|
| 68 |
+
"reserved_bytes_current_delta": 25513951232,
|
| 69 |
+
"reserved_bytes_peak": 128687538176,
|
| 70 |
+
"reserved_bytes_peak_delta": 25513951232,
|
| 71 |
+
"segment_count_current": 2005,
|
| 72 |
+
"segment_count_current_delta": 219,
|
| 73 |
+
"segment_count_peak": 2005,
|
| 74 |
+
"segment_count_peak_delta": 219,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:35.595543Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6c2ab06c8c945ecdb672016c8f54cb38720c97c1653b13de54badb3b299e51a4
|
| 3 |
+
size 2586991
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85292954112,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121477075968,
|
| 5 |
+
"active_bytes_peak_delta": 49152,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85292954112,
|
| 8 |
+
"active_bytes_peak": 121477075968,
|
| 9 |
+
"allocated_bytes_current": 85292954112,
|
| 10 |
+
"allocated_bytes_peak": 121477075968,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1770311168,
|
| 13 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 128687538176,
|
| 21 |
+
"reserved_bytes_peak": 128687538176,
|
| 22 |
+
"segment_count_current": 2005,
|
| 23 |
+
"segment_count_peak": 2005,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.971739Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85292954112,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121477075968,
|
| 30 |
+
"allocated_bytes_peak_delta": 49152,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85292954112,
|
| 33 |
+
"active_bytes_peak": 121477026816,
|
| 34 |
+
"allocated_bytes_current": 85292954112,
|
| 35 |
+
"allocated_bytes_peak": 121477026816,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1770311168,
|
| 38 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 128687538176,
|
| 46 |
+
"reserved_bytes_peak": 128687538176,
|
| 47 |
+
"segment_count_current": 2005,
|
| 48 |
+
"segment_count_peak": 2005,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:35.601397Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1770311168,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128687538176,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 128687538176,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2005,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2005,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:36.017227Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1e91cae5e5b40799ac6bd86fc41b310aa28ee2e4b11be02b9bd4055f177b4cb5
|
| 3 |
+
size 2609276
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85309997568,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 128733144064,
|
| 5 |
+
"active_bytes_peak_delta": 7256068096,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85309997568,
|
| 8 |
+
"active_bytes_peak": 128733144064,
|
| 9 |
+
"allocated_bytes_current": 85309997568,
|
| 10 |
+
"allocated_bytes_peak": 121485386752,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1453374976,
|
| 13 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 141482262528,
|
| 21 |
+
"reserved_bytes_peak": 141482262528,
|
| 22 |
+
"segment_count_current": 1805,
|
| 23 |
+
"segment_count_peak": 2005,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:38.956995Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85309997568,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 121485386752,
|
| 30 |
+
"allocated_bytes_peak_delta": 8310784,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85292954112,
|
| 33 |
+
"active_bytes_peak": 121477075968,
|
| 34 |
+
"allocated_bytes_current": 85292954112,
|
| 35 |
+
"allocated_bytes_peak": 121477075968,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1770311168,
|
| 38 |
+
"inactive_split_bytes_peak": 11201188352,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 128687538176,
|
| 46 |
+
"reserved_bytes_peak": 128687538176,
|
| 47 |
+
"segment_count_current": 2005,
|
| 48 |
+
"segment_count_peak": 2005,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:36.022500Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1453374976,
|
| 55 |
+
"inactive_split_bytes_current_delta": -316936192,
|
| 56 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1681393152,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141482262528,
|
| 68 |
+
"reserved_bytes_current_delta": 12794724352,
|
| 69 |
+
"reserved_bytes_peak": 141482262528,
|
| 70 |
+
"reserved_bytes_peak_delta": 12794724352,
|
| 71 |
+
"segment_count_current": 1805,
|
| 72 |
+
"segment_count_current_delta": -200,
|
| 73 |
+
"segment_count_peak": 2005,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.010451Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e3a5548b896b2a25ca4a2fb6b2899d86e34f93bf80fffcbbb54656f939a519c
|
| 3 |
+
size 2779026
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85309997568,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128733144064,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85309997568,
|
| 8 |
+
"active_bytes_peak": 128733144064,
|
| 9 |
+
"allocated_bytes_current": 85309997568,
|
| 10 |
+
"allocated_bytes_peak": 121485386752,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1453374976,
|
| 13 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 141482262528,
|
| 21 |
+
"reserved_bytes_peak": 141482262528,
|
| 22 |
+
"segment_count_current": 1805,
|
| 23 |
+
"segment_count_peak": 2005,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.331583Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85309997568,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121485386752,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85309997568,
|
| 33 |
+
"active_bytes_peak": 128733144064,
|
| 34 |
+
"allocated_bytes_current": 85309997568,
|
| 35 |
+
"allocated_bytes_peak": 121485386752,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1453374976,
|
| 38 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 141482262528,
|
| 46 |
+
"reserved_bytes_peak": 141482262528,
|
| 47 |
+
"segment_count_current": 1805,
|
| 48 |
+
"segment_count_peak": 2005,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.017489Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1453374976,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141482262528,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141482262528,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1805,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2005,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.342045Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6183e257ea248a054b02716706577af577188c192e256577c3ffa6d94ccb4de7
|
| 3 |
+
size 422083
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85309997568,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128733144064,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85309997568,
|
| 8 |
+
"active_bytes_peak": 128733144064,
|
| 9 |
+
"allocated_bytes_current": 85309997568,
|
| 10 |
+
"allocated_bytes_peak": 121485386752,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1453374976,
|
| 13 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 141482262528,
|
| 21 |
+
"reserved_bytes_peak": 141482262528,
|
| 22 |
+
"segment_count_current": 1805,
|
| 23 |
+
"segment_count_peak": 2005,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.652667Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85309997568,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121485386752,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85309997568,
|
| 33 |
+
"active_bytes_peak": 128733144064,
|
| 34 |
+
"allocated_bytes_current": 85309997568,
|
| 35 |
+
"allocated_bytes_peak": 121485386752,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1453374976,
|
| 38 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 141482262528,
|
| 46 |
+
"reserved_bytes_peak": 141482262528,
|
| 47 |
+
"segment_count_current": 1805,
|
| 48 |
+
"segment_count_peak": 2005,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.343193Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1453374976,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12882581504,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141482262528,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141482262528,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1805,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2005,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.662359Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7bd61744ce6d87e4c745b1d183902c4d00ccc1bd8a56aad6e8a91430ad96fbb6
|
| 3 |
+
size 422083
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 40774371840,
|
| 4 |
+
"active_bytes_peak": 90612776448,
|
| 5 |
+
"active_bytes_peak_delta": 46067000832,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 90612776448,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 90612776448,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1566957056,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 103244890112,
|
| 21 |
+
"reserved_bytes_peak": 103244890112,
|
| 22 |
+
"segment_count_current": 1790,
|
| 23 |
+
"segment_count_peak": 1790,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:34.927803Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 40774371840,
|
| 29 |
+
"allocated_bytes_peak": 90612776448,
|
| 30 |
+
"allocated_bytes_peak_delta": 46067000832,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 421357568,
|
| 38 |
+
"inactive_split_bytes_peak": 1025337344,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:49:54.861521Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1566957056,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1145599488,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10888034816,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103244890112,
|
| 68 |
+
"reserved_bytes_current_delta": 54993616896,
|
| 69 |
+
"reserved_bytes_peak": 103244890112,
|
| 70 |
+
"reserved_bytes_peak_delta": 54993616896,
|
| 71 |
+
"segment_count_current": 1790,
|
| 72 |
+
"segment_count_current_delta": 1169,
|
| 73 |
+
"segment_count_peak": 1790,
|
| 74 |
+
"segment_count_peak_delta": 1169,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:34.980104Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bd97ae4b71d75d8df2690b1b9bd0903c3090a75cbc027e10b301f7c4dccd2865
|
| 3 |
+
size 3148002
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121478210048,
|
| 5 |
+
"active_bytes_peak_delta": 30865433600,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 121478210048,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 121478210048,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1969610240,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 128804978688,
|
| 21 |
+
"reserved_bytes_peak": 128804978688,
|
| 22 |
+
"segment_count_current": 2012,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.546159Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121478210048,
|
| 30 |
+
"allocated_bytes_peak_delta": 30865433600,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 90612776448,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 90612776448,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1566957056,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 103244890112,
|
| 46 |
+
"reserved_bytes_peak": 103244890112,
|
| 47 |
+
"segment_count_current": 1790,
|
| 48 |
+
"segment_count_peak": 1790,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:34.986365Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1969610240,
|
| 55 |
+
"inactive_split_bytes_current_delta": 402653184,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128804978688,
|
| 68 |
+
"reserved_bytes_current_delta": 25560088576,
|
| 69 |
+
"reserved_bytes_peak": 128804978688,
|
| 70 |
+
"reserved_bytes_peak_delta": 25560088576,
|
| 71 |
+
"segment_count_current": 2012,
|
| 72 |
+
"segment_count_current_delta": 222,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 222,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:35.596037Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:af77a838b6c6a61374c9704ed59f260069beb78734595ba0949d7d8f173b1c0b
|
| 3 |
+
size 2579979
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121478455808,
|
| 5 |
+
"active_bytes_peak_delta": 245760,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 121478455808,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 121478455808,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1971707392,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 128804978688,
|
| 21 |
+
"reserved_bytes_peak": 128804978688,
|
| 22 |
+
"segment_count_current": 2012,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.971660Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121478455808,
|
| 30 |
+
"allocated_bytes_peak_delta": 245760,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 121478210048,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 121478210048,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1969610240,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 128804978688,
|
| 46 |
+
"reserved_bytes_peak": 128804978688,
|
| 47 |
+
"segment_count_current": 2012,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:35.602405Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1971707392,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128804978688,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 128804978688,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2012,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:36.017792Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8705912aae623c560ae0d7d576d6e67e34ff869f0e694974c17f0d2cccc07fd0
|
| 3 |
+
size 2569019
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 7256715776,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:38.956984Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 8958464,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 121478455808,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 121478455808,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1971707392,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 128804978688,
|
| 46 |
+
"reserved_bytes_peak": 128804978688,
|
| 47 |
+
"segment_count_current": 2012,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:36.022865Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": -319033344,
|
| 56 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1170605568,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 12880707584,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 12880707584,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": -197,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.010026Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:29bc1811482194f01ba57189d5b0336f0a556b13bcd06f639573a807a68f2d02
|
| 3 |
+
size 2771672
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.331461Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312025088,
|
| 33 |
+
"active_bytes_peak": 128735171584,
|
| 34 |
+
"allocated_bytes_current": 85312025088,
|
| 35 |
+
"allocated_bytes_peak": 121487414272,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1652674048,
|
| 38 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 141685686272,
|
| 46 |
+
"reserved_bytes_peak": 141685686272,
|
| 47 |
+
"segment_count_current": 1815,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.015933Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.341247Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f31fa3e47c17b389551300bd90296f5b7d4a52bd842eebe30fe68356467f07a0
|
| 3 |
+
size 416145
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.652649Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312025088,
|
| 33 |
+
"active_bytes_peak": 128735171584,
|
| 34 |
+
"allocated_bytes_current": 85312025088,
|
| 35 |
+
"allocated_bytes_peak": 121487414272,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1652674048,
|
| 38 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 141685686272,
|
| 46 |
+
"reserved_bytes_peak": 141685686272,
|
| 47 |
+
"segment_count_current": 1815,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.342351Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.662081Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:52f33d0f617a93d8d768f32da49c5a936943607e881cc1a361ba3301d7d47d13
|
| 3 |
+
size 416145
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 40774371840,
|
| 4 |
+
"active_bytes_peak": 90612776448,
|
| 5 |
+
"active_bytes_peak_delta": 46067000832,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 90612776448,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 90612776448,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1566957056,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 103244890112,
|
| 21 |
+
"reserved_bytes_peak": 103244890112,
|
| 22 |
+
"segment_count_current": 1790,
|
| 23 |
+
"segment_count_peak": 1790,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:34.927806Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 40774371840,
|
| 29 |
+
"allocated_bytes_peak": 90612776448,
|
| 30 |
+
"allocated_bytes_peak_delta": 46067000832,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 421357568,
|
| 38 |
+
"inactive_split_bytes_peak": 1025337344,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:49:54.861483Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1566957056,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1145599488,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10888034816,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103244890112,
|
| 68 |
+
"reserved_bytes_current_delta": 54993616896,
|
| 69 |
+
"reserved_bytes_peak": 103244890112,
|
| 70 |
+
"reserved_bytes_peak_delta": 54993616896,
|
| 71 |
+
"segment_count_current": 1790,
|
| 72 |
+
"segment_count_current_delta": 1169,
|
| 73 |
+
"segment_count_peak": 1790,
|
| 74 |
+
"segment_count_peak_delta": 1169,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:34.981556Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:27b146b2dfaeba3efc68604c5b3c53fb4d0bbb10f8c3a1b64518f0b3c3060ef1
|
| 3 |
+
size 3148002
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121478210048,
|
| 5 |
+
"active_bytes_peak_delta": 30865433600,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 121478210048,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 121478210048,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1969610240,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 128804978688,
|
| 21 |
+
"reserved_bytes_peak": 128804978688,
|
| 22 |
+
"segment_count_current": 2012,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.546227Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121478210048,
|
| 30 |
+
"allocated_bytes_peak_delta": 30865433600,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 90612776448,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 90612776448,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1566957056,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 103244890112,
|
| 46 |
+
"reserved_bytes_peak": 103244890112,
|
| 47 |
+
"segment_count_current": 1790,
|
| 48 |
+
"segment_count_peak": 1790,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:34.987886Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1969610240,
|
| 55 |
+
"inactive_split_bytes_current_delta": 402653184,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128804978688,
|
| 68 |
+
"reserved_bytes_current_delta": 25560088576,
|
| 69 |
+
"reserved_bytes_peak": 128804978688,
|
| 70 |
+
"reserved_bytes_peak_delta": 25560088576,
|
| 71 |
+
"segment_count_current": 2012,
|
| 72 |
+
"segment_count_current_delta": 222,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 222,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:35.595565Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c5b1e78a390fe971af9f4fc30d95e219aac098eba527b9f930025a3321f93456
|
| 3 |
+
size 2579948
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121478455808,
|
| 5 |
+
"active_bytes_peak_delta": 245760,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 121478455808,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 121478455808,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1971707392,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 128804978688,
|
| 21 |
+
"reserved_bytes_peak": 128804978688,
|
| 22 |
+
"segment_count_current": 2012,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.971658Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121478455808,
|
| 30 |
+
"allocated_bytes_peak_delta": 245760,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 121478210048,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 121478210048,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1969610240,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 128804978688,
|
| 46 |
+
"reserved_bytes_peak": 128804978688,
|
| 47 |
+
"segment_count_current": 2012,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:35.601963Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1971707392,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128804978688,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 128804978688,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2012,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:36.016491Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:249e9b8f6d474565c05df97910be1857323ef733f2bf81f86f2e60db2e0c0a4f
|
| 3 |
+
size 2569055
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 7256715776,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:38.956987Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 8958464,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 121478455808,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 121478455808,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1971707392,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 128804978688,
|
| 46 |
+
"reserved_bytes_peak": 128804978688,
|
| 47 |
+
"segment_count_current": 2012,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:36.021547Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": -319033344,
|
| 56 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1170605568,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 12880707584,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 12880707584,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": -197,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.010224Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4861d2922ba964c05d12e06573ce96afcd744654d2763ee57e9b0332af50b0e9
|
| 3 |
+
size 2771672
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.331456Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312025088,
|
| 33 |
+
"active_bytes_peak": 128735171584,
|
| 34 |
+
"allocated_bytes_current": 85312025088,
|
| 35 |
+
"allocated_bytes_peak": 121487414272,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1652674048,
|
| 38 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 141685686272,
|
| 46 |
+
"reserved_bytes_peak": 141685686272,
|
| 47 |
+
"segment_count_current": 1815,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.016318Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.341347Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:91b5d7b1733ad80c6a16d831fe2407c6b08974fd6fc2484f9afd62e0a6f0c9d0
|
| 3 |
+
size 416145
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.652654Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312025088,
|
| 33 |
+
"active_bytes_peak": 128735171584,
|
| 34 |
+
"allocated_bytes_current": 85312025088,
|
| 35 |
+
"allocated_bytes_peak": 121487414272,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1652674048,
|
| 38 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 141685686272,
|
| 46 |
+
"reserved_bytes_peak": 141685686272,
|
| 47 |
+
"segment_count_current": 1815,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.342440Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13083977728,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.662013Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:48bea32c141f6c0dc478293871003f1d61d30ebf5a6daf2e6748d169924c2102
|
| 3 |
+
size 416145
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 40774371840,
|
| 4 |
+
"active_bytes_peak": 90612776448,
|
| 5 |
+
"active_bytes_peak_delta": 46067000832,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 90612776448,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 90612776448,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1566957056,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 103244890112,
|
| 21 |
+
"reserved_bytes_peak": 103244890112,
|
| 22 |
+
"segment_count_current": 1790,
|
| 23 |
+
"segment_count_peak": 1790,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:34.927781Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 40774371840,
|
| 29 |
+
"allocated_bytes_peak": 90612776448,
|
| 30 |
+
"allocated_bytes_peak_delta": 46067000832,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 421357568,
|
| 38 |
+
"inactive_split_bytes_peak": 1025337344,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:49:54.861469Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1566957056,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1145599488,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10888034816,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103244890112,
|
| 68 |
+
"reserved_bytes_current_delta": 54993616896,
|
| 69 |
+
"reserved_bytes_peak": 103244890112,
|
| 70 |
+
"reserved_bytes_peak_delta": 54993616896,
|
| 71 |
+
"segment_count_current": 1790,
|
| 72 |
+
"segment_count_current_delta": 1169,
|
| 73 |
+
"segment_count_peak": 1790,
|
| 74 |
+
"segment_count_peak_delta": 1169,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:34.979803Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5bd36093f315ca9b352532172956de6dfd4183b9176483c7a47e15c64fba50af
|
| 3 |
+
size 3147999
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121478210048,
|
| 5 |
+
"active_bytes_peak_delta": 30865433600,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 121478210048,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 121478210048,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1969610240,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 128804978688,
|
| 21 |
+
"reserved_bytes_peak": 128804978688,
|
| 22 |
+
"segment_count_current": 2012,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.546147Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121478210048,
|
| 30 |
+
"allocated_bytes_peak_delta": 30865433600,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 90612776448,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 90612776448,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1566957056,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 103244890112,
|
| 46 |
+
"reserved_bytes_peak": 103244890112,
|
| 47 |
+
"segment_count_current": 1790,
|
| 48 |
+
"segment_count_peak": 1790,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:34.986030Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1969610240,
|
| 55 |
+
"inactive_split_bytes_current_delta": 402653184,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128804978688,
|
| 68 |
+
"reserved_bytes_current_delta": 25560088576,
|
| 69 |
+
"reserved_bytes_peak": 128804978688,
|
| 70 |
+
"reserved_bytes_peak_delta": 25560088576,
|
| 71 |
+
"segment_count_current": 2012,
|
| 72 |
+
"segment_count_current_delta": 222,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 222,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:35.594867Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e587809f83fdda618266730bd28e2282bd753e40bc02169039a8e4b34ceaf25d
|
| 3 |
+
size 2579979
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85294981632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122082435584,
|
| 5 |
+
"active_bytes_peak_delta": 604225536,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85294981632,
|
| 8 |
+
"active_bytes_peak": 122082435584,
|
| 9 |
+
"allocated_bytes_current": 85294981632,
|
| 10 |
+
"allocated_bytes_peak": 121478455808,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1971707392,
|
| 13 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 128804978688,
|
| 21 |
+
"reserved_bytes_peak": 128804978688,
|
| 22 |
+
"segment_count_current": 2012,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:35.971654Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85294981632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121478455808,
|
| 30 |
+
"allocated_bytes_peak_delta": 245760,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 121478210048,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 121478210048,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1969610240,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 128804978688,
|
| 46 |
+
"reserved_bytes_peak": 128804978688,
|
| 47 |
+
"segment_count_current": 2012,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:35.601253Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1971707392,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 128804978688,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 128804978688,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2012,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:36.016411Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ebc496a7c1c0b713c075ec11ebed6a4aca0be3a6c15bf3abbe78aafc67ca8b8
|
| 3 |
+
size 2569019
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 6652736000,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:38.956989Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 8958464,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85294981632,
|
| 33 |
+
"active_bytes_peak": 122082435584,
|
| 34 |
+
"allocated_bytes_current": 85294981632,
|
| 35 |
+
"allocated_bytes_peak": 121478455808,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1971707392,
|
| 38 |
+
"inactive_split_bytes_peak": 11913372160,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 128804978688,
|
| 46 |
+
"reserved_bytes_peak": 128804978688,
|
| 47 |
+
"segment_count_current": 2012,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:36.021568Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": -319033344,
|
| 56 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 902170112,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 12880707584,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 12880707584,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": -197,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.010626Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e19767dc90ece8d439f6886a341dd416885067ba2615502ae69deab56959ca22
|
| 3 |
+
size 2771669
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.331469Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312025088,
|
| 33 |
+
"active_bytes_peak": 128735171584,
|
| 34 |
+
"allocated_bytes_current": 85312025088,
|
| 35 |
+
"allocated_bytes_peak": 121487414272,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1652674048,
|
| 38 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 141685686272,
|
| 46 |
+
"reserved_bytes_peak": 141685686272,
|
| 47 |
+
"segment_count_current": 1815,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.016682Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.341608Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7ba49380f0f1c7c224d6180fa854e53bfa52c31ea95d95b166dfceece6a8f2e2
|
| 3 |
+
size 416035
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312025088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 128735171584,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312025088,
|
| 8 |
+
"active_bytes_peak": 128735171584,
|
| 9 |
+
"allocated_bytes_current": 85312025088,
|
| 10 |
+
"allocated_bytes_peak": 121487414272,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1652674048,
|
| 13 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 141685686272,
|
| 21 |
+
"reserved_bytes_peak": 141685686272,
|
| 22 |
+
"segment_count_current": 1815,
|
| 23 |
+
"segment_count_peak": 2012,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:39.652658Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312025088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121487414272,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312025088,
|
| 33 |
+
"active_bytes_peak": 128735171584,
|
| 34 |
+
"allocated_bytes_current": 85312025088,
|
| 35 |
+
"allocated_bytes_peak": 121487414272,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1652674048,
|
| 38 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 141685686272,
|
| 46 |
+
"reserved_bytes_peak": 141685686272,
|
| 47 |
+
"segment_count_current": 1815,
|
| 48 |
+
"segment_count_peak": 2012,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:50:39.342717Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1652674048,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12815542272,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 141685686272,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 141685686272,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1815,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2012,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:39.662417Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:77688a743792e2b71ed5652b03f4fb807657b5a012cf454a3c1eceb47d63c720
|
| 3 |
+
size 416035
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295019520,
|
| 3 |
+
"active_bytes_current_delta": 40774409728,
|
| 4 |
+
"active_bytes_peak": 90623401472,
|
| 5 |
+
"active_bytes_peak_delta": 46077625856,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295019520,
|
| 8 |
+
"active_bytes_peak": 90623401472,
|
| 9 |
+
"allocated_bytes_current": 85295019520,
|
| 10 |
+
"allocated_bytes_peak": 90623401472,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1720011264,
|
| 13 |
+
"inactive_split_bytes_peak": 11677395968,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 103291027456,
|
| 21 |
+
"reserved_bytes_peak": 103291027456,
|
| 22 |
+
"segment_count_current": 1798,
|
| 23 |
+
"segment_count_peak": 1798,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:50:34.927754Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295019520,
|
| 28 |
+
"allocated_bytes_current_delta": 40774409728,
|
| 29 |
+
"allocated_bytes_peak": 90623401472,
|
| 30 |
+
"allocated_bytes_peak_delta": 46077625856,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 270362624,
|
| 38 |
+
"inactive_split_bytes_peak": 874342400,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 47949283328,
|
| 46 |
+
"reserved_bytes_peak": 47949283328,
|
| 47 |
+
"segment_count_current": 615,
|
| 48 |
+
"segment_count_peak": 615,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:49:54.861438Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1720011264,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1449648640,
|
| 56 |
+
"inactive_split_bytes_peak": 11677395968,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10803053568,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103291027456,
|
| 68 |
+
"reserved_bytes_current_delta": 55341744128,
|
| 69 |
+
"reserved_bytes_peak": 103291027456,
|
| 70 |
+
"reserved_bytes_peak_delta": 55341744128,
|
| 71 |
+
"segment_count_current": 1798,
|
| 72 |
+
"segment_count_current_delta": 1183,
|
| 73 |
+
"segment_count_peak": 1798,
|
| 74 |
+
"segment_count_peak_delta": 1183,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:50:34.981424Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|