granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +152 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env +61 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log +0 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +176 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle +3 -0
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 4
|
| 43 |
+
2, 4
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 24
|
| 49 |
+
0, 899
|
| 50 |
+
1, 899
|
| 51 |
+
2, 901
|
| 52 |
+
3, 913
|
| 53 |
+
4, 903
|
| 54 |
+
5, 909
|
| 55 |
+
6, 909
|
| 56 |
+
7, 901
|
| 57 |
+
0, 1955
|
| 58 |
+
1, 1955
|
| 59 |
+
2, 1955
|
| 60 |
+
3, 1955
|
| 61 |
+
4, 1955
|
| 62 |
+
5, 1955
|
| 63 |
+
6, 1955
|
| 64 |
+
7, 1955
|
| 65 |
+
0, 2411
|
| 66 |
+
1, 2405
|
| 67 |
+
2, 2413
|
| 68 |
+
3, 2409
|
| 69 |
+
4, 2417
|
| 70 |
+
5, 2427
|
| 71 |
+
6, 2417
|
| 72 |
+
7, 2413
|
| 73 |
+
0, 25239
|
| 74 |
+
1, 25239
|
| 75 |
+
2, 25239
|
| 76 |
+
3, 25239
|
| 77 |
+
4, 25239
|
| 78 |
+
5, 25239
|
| 79 |
+
6, 25239
|
| 80 |
+
7, 25239
|
| 81 |
+
0, 28863
|
| 82 |
+
1, 28929
|
| 83 |
+
2, 28773
|
| 84 |
+
3, 28787
|
| 85 |
+
4, 28781
|
| 86 |
+
5, 28761
|
| 87 |
+
6, 28915
|
| 88 |
+
7, 28997
|
| 89 |
+
0, 38975
|
| 90 |
+
1, 39047
|
| 91 |
+
2, 38905
|
| 92 |
+
3, 38869
|
| 93 |
+
4, 38897
|
| 94 |
+
5, 38895
|
| 95 |
+
6, 39009
|
| 96 |
+
7, 39049
|
| 97 |
+
0, 39773
|
| 98 |
+
1, 39857
|
| 99 |
+
2, 39707
|
| 100 |
+
3, 39705
|
| 101 |
+
4, 39705
|
| 102 |
+
5, 39703
|
| 103 |
+
6, 39839
|
| 104 |
+
7, 39857
|
| 105 |
+
0, 39773
|
| 106 |
+
1, 39857
|
| 107 |
+
2, 39707
|
| 108 |
+
3, 39705
|
| 109 |
+
4, 39705
|
| 110 |
+
5, 39703
|
| 111 |
+
6, 39839
|
| 112 |
+
7, 39857
|
| 113 |
+
0, 39783
|
| 114 |
+
1, 39867
|
| 115 |
+
2, 39717
|
| 116 |
+
3, 39717
|
| 117 |
+
4, 39717
|
| 118 |
+
5, 39713
|
| 119 |
+
6, 39849
|
| 120 |
+
7, 39867
|
| 121 |
+
0, 39783
|
| 122 |
+
1, 39867
|
| 123 |
+
2, 39717
|
| 124 |
+
3, 39717
|
| 125 |
+
4, 39717
|
| 126 |
+
5, 39713
|
| 127 |
+
6, 39849
|
| 128 |
+
7, 39867
|
| 129 |
+
0, 109499
|
| 130 |
+
1, 109597
|
| 131 |
+
2, 109441
|
| 132 |
+
3, 109445
|
| 133 |
+
4, 109451
|
| 134 |
+
5, 109447
|
| 135 |
+
6, 109573
|
| 136 |
+
7, 109607
|
| 137 |
+
0, 110101
|
| 138 |
+
1, 110185
|
| 139 |
+
2, 110035
|
| 140 |
+
3, 110035
|
| 141 |
+
4, 110035
|
| 142 |
+
5, 110031
|
| 143 |
+
6, 110167
|
| 144 |
+
7, 110185
|
| 145 |
+
0, 110105
|
| 146 |
+
1, 110191
|
| 147 |
+
2, 110041
|
| 148 |
+
3, 110039
|
| 149 |
+
4, 110039
|
| 150 |
+
5, 110037
|
| 151 |
+
6, 110171
|
| 152 |
+
7, 110191
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=DSV2_LITE_L24
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-better-memory-dump-support
|
| 6 |
+
commit=c3a8a8a7
|
| 7 |
+
num_layers=24
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=10944
|
| 10 |
+
moe_ffn_hidden=1408
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=6
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=8650752
|
| 15 |
+
heads=16
|
| 16 |
+
kv_groups=0
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=2816
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=1
|
| 22 |
+
dense_first_layers=1
|
| 23 |
+
moe_layers=23
|
| 24 |
+
rotary_base=10000
|
| 25 |
+
rotary_scaling=40
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=seq_aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=49152
|
| 39 |
+
mean_tokens_per_expert=6144
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.00
|
| 44 |
+
cuda_graph=1
|
| 45 |
+
probe_mode=0
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=memory
|
| 48 |
+
memory_dump_mode=iteration
|
| 49 |
+
memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=6
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=768
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv
CHANGED
|
@@ -22,3 +22,179 @@
|
|
| 22 |
5, 1
|
| 23 |
6, 1
|
| 24 |
7, 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
5, 1
|
| 23 |
6, 1
|
| 24 |
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 27
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 821
|
| 42 |
+
1, 819
|
| 43 |
+
2, 819
|
| 44 |
+
3, 819
|
| 45 |
+
4, 819
|
| 46 |
+
5, 819
|
| 47 |
+
6, 819
|
| 48 |
+
7, 819
|
| 49 |
+
0, 1537
|
| 50 |
+
1, 1537
|
| 51 |
+
2, 1537
|
| 52 |
+
3, 1537
|
| 53 |
+
4, 1537
|
| 54 |
+
5, 1537
|
| 55 |
+
6, 1537
|
| 56 |
+
7, 1537
|
| 57 |
+
0, 2551
|
| 58 |
+
1, 2551
|
| 59 |
+
2, 2551
|
| 60 |
+
3, 2551
|
| 61 |
+
4, 2551
|
| 62 |
+
5, 2551
|
| 63 |
+
6, 2551
|
| 64 |
+
7, 2551
|
| 65 |
+
0, 2959
|
| 66 |
+
1, 2959
|
| 67 |
+
2, 2959
|
| 68 |
+
3, 2959
|
| 69 |
+
4, 2959
|
| 70 |
+
5, 2959
|
| 71 |
+
6, 2959
|
| 72 |
+
7, 2959
|
| 73 |
+
0, 51537
|
| 74 |
+
1, 51537
|
| 75 |
+
2, 51537
|
| 76 |
+
3, 51537
|
| 77 |
+
4, 51249
|
| 78 |
+
5, 51537
|
| 79 |
+
6, 51537
|
| 80 |
+
7, 51537
|
| 81 |
+
0, 53019
|
| 82 |
+
1, 53019
|
| 83 |
+
2, 53017
|
| 84 |
+
3, 53019
|
| 85 |
+
4, 52729
|
| 86 |
+
5, 53019
|
| 87 |
+
6, 53017
|
| 88 |
+
7, 53019
|
| 89 |
+
0, 54651
|
| 90 |
+
1, 54651
|
| 91 |
+
2, 54651
|
| 92 |
+
3, 54651
|
| 93 |
+
4, 54363
|
| 94 |
+
5, 54651
|
| 95 |
+
6, 54651
|
| 96 |
+
7, 54651
|
| 97 |
+
0, 55163
|
| 98 |
+
1, 55163
|
| 99 |
+
2, 55163
|
| 100 |
+
3, 55163
|
| 101 |
+
4, 54875
|
| 102 |
+
5, 55163
|
| 103 |
+
6, 55163
|
| 104 |
+
7, 55163
|
| 105 |
+
0, 55163
|
| 106 |
+
1, 55163
|
| 107 |
+
2, 55163
|
| 108 |
+
3, 55163
|
| 109 |
+
4, 54875
|
| 110 |
+
5, 55163
|
| 111 |
+
6, 55163
|
| 112 |
+
7, 55163
|
| 113 |
+
0, 56699
|
| 114 |
+
1, 56699
|
| 115 |
+
2, 56699
|
| 116 |
+
3, 56699
|
| 117 |
+
4, 56411
|
| 118 |
+
5, 56699
|
| 119 |
+
6, 56699
|
| 120 |
+
7, 56699
|
| 121 |
+
0, 87805
|
| 122 |
+
1, 87845
|
| 123 |
+
2, 87845
|
| 124 |
+
3, 87845
|
| 125 |
+
4, 87841
|
| 126 |
+
5, 87845
|
| 127 |
+
6, 87845
|
| 128 |
+
7, 87845
|
| 129 |
+
0, 88517
|
| 130 |
+
1, 88557
|
| 131 |
+
2, 88557
|
| 132 |
+
3, 88557
|
| 133 |
+
4, 88553
|
| 134 |
+
5, 88557
|
| 135 |
+
6, 88557
|
| 136 |
+
7, 88557
|
| 137 |
+
0, 88517
|
| 138 |
+
1, 88557
|
| 139 |
+
2, 88557
|
| 140 |
+
3, 88557
|
| 141 |
+
4, 88553
|
| 142 |
+
5, 88557
|
| 143 |
+
6, 88557
|
| 144 |
+
7, 88557
|
| 145 |
+
0, 88517
|
| 146 |
+
1, 88557
|
| 147 |
+
2, 88557
|
| 148 |
+
3, 88557
|
| 149 |
+
4, 88553
|
| 150 |
+
5, 88557
|
| 151 |
+
6, 88557
|
| 152 |
+
7, 88557
|
| 153 |
+
0, 88921
|
| 154 |
+
1, 88961
|
| 155 |
+
2, 88961
|
| 156 |
+
3, 88961
|
| 157 |
+
4, 88957
|
| 158 |
+
5, 88961
|
| 159 |
+
6, 88961
|
| 160 |
+
7, 88961
|
| 161 |
+
0, 95725
|
| 162 |
+
1, 95789
|
| 163 |
+
2, 95721
|
| 164 |
+
3, 95743
|
| 165 |
+
4, 95923
|
| 166 |
+
5, 95779
|
| 167 |
+
6, 95741
|
| 168 |
+
7, 95759
|
| 169 |
+
0, 111083
|
| 170 |
+
1, 113027
|
| 171 |
+
2, 114123
|
| 172 |
+
3, 111903
|
| 173 |
+
4, 110463
|
| 174 |
+
5, 116199
|
| 175 |
+
6, 108963
|
| 176 |
+
7, 111075
|
| 177 |
+
0, 89083
|
| 178 |
+
1, 89237
|
| 179 |
+
2, 87617
|
| 180 |
+
3, 92987
|
| 181 |
+
4, 91423
|
| 182 |
+
5, 90119
|
| 183 |
+
6, 90181
|
| 184 |
+
7, 88643
|
| 185 |
+
0, 121025
|
| 186 |
+
1, 121181
|
| 187 |
+
2, 121181
|
| 188 |
+
3, 121181
|
| 189 |
+
4, 121321
|
| 190 |
+
5, 121181
|
| 191 |
+
6, 121181
|
| 192 |
+
7, 121181
|
| 193 |
+
0, 5275
|
| 194 |
+
1, 3678
|
| 195 |
+
2, 3678
|
| 196 |
+
3, 3678
|
| 197 |
+
4, 3678
|
| 198 |
+
5, 3678
|
| 199 |
+
6, 3998
|
| 200 |
+
7, 121179
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77298383360,
|
| 3 |
+
"active_bytes_current_delta": 32777583104,
|
| 4 |
+
"active_bytes_peak": 82477897216,
|
| 5 |
+
"active_bytes_peak_delta": 37931931136,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77298383360,
|
| 8 |
+
"active_bytes_peak": 82477897216,
|
| 9 |
+
"allocated_bytes_current": 77298383360,
|
| 10 |
+
"allocated_bytes_peak": 82477897216,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1206404608,
|
| 13 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 90250936320,
|
| 21 |
+
"reserved_bytes_peak": 90250936320,
|
| 22 |
+
"segment_count_current": 1714,
|
| 23 |
+
"segment_count_peak": 1714,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:43.968857Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77298383360,
|
| 28 |
+
"allocated_bytes_current_delta": 32777583104,
|
| 29 |
+
"allocated_bytes_peak": 82477897216,
|
| 30 |
+
"allocated_bytes_peak_delta": 37931931136,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 446332928,
|
| 38 |
+
"inactive_split_bytes_peak": 1050312704,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:17.937284Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1206404608,
|
| 55 |
+
"inactive_split_bytes_current_delta": 760071680,
|
| 56 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 8311884288,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90250936320,
|
| 68 |
+
"reserved_bytes_current_delta": 41999663104,
|
| 69 |
+
"reserved_bytes_peak": 90250936320,
|
| 70 |
+
"reserved_bytes_peak_delta": 41999663104,
|
| 71 |
+
"segment_count_current": 1714,
|
| 72 |
+
"segment_count_current_delta": 1093,
|
| 73 |
+
"segment_count_peak": 1714,
|
| 74 |
+
"segment_count_peak_delta": 1093,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.009882Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:366506f1eecdad1148894ecfb8f4a0105384eba9793cdc997f78dc0f25a97f89
|
| 3 |
+
size 2316617
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77298383360,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113343831040,
|
| 5 |
+
"active_bytes_peak_delta": 30865933824,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77298383360,
|
| 8 |
+
"active_bytes_peak": 113343831040,
|
| 9 |
+
"allocated_bytes_current": 77298383360,
|
| 10 |
+
"allocated_bytes_peak": 113343831040,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1206404608,
|
| 13 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 115242696704,
|
| 21 |
+
"reserved_bytes_peak": 115242696704,
|
| 22 |
+
"segment_count_current": 1923,
|
| 23 |
+
"segment_count_peak": 1923,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:44.855042Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77298383360,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113343831040,
|
| 30 |
+
"allocated_bytes_peak_delta": 30865933824,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77298383360,
|
| 33 |
+
"active_bytes_peak": 82477897216,
|
| 34 |
+
"allocated_bytes_current": 77298383360,
|
| 35 |
+
"allocated_bytes_peak": 82477897216,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1206404608,
|
| 38 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 90250936320,
|
| 46 |
+
"reserved_bytes_peak": 90250936320,
|
| 47 |
+
"segment_count_current": 1714,
|
| 48 |
+
"segment_count_peak": 1714,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.014581Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1206404608,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115242696704,
|
| 68 |
+
"reserved_bytes_current_delta": 24991760384,
|
| 69 |
+
"reserved_bytes_peak": 115242696704,
|
| 70 |
+
"reserved_bytes_peak_delta": 24991760384,
|
| 71 |
+
"segment_count_current": 1923,
|
| 72 |
+
"segment_count_current_delta": 209,
|
| 73 |
+
"segment_count_peak": 1923,
|
| 74 |
+
"segment_count_peak_delta": 209,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.891762Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a9d644fd17905670a38a6b94b76cb0b9e6d9eeb7d69c1392488c0adbc3ebceca
|
| 3 |
+
size 1844220
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77298383360,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113343831040,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77298383360,
|
| 8 |
+
"active_bytes_peak": 113343831040,
|
| 9 |
+
"allocated_bytes_current": 77298383360,
|
| 10 |
+
"allocated_bytes_peak": 113343831040,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1206404608,
|
| 13 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 115242696704,
|
| 21 |
+
"reserved_bytes_peak": 115242696704,
|
| 22 |
+
"segment_count_current": 1923,
|
| 23 |
+
"segment_count_peak": 1923,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:45.209078Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77298383360,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113343831040,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77298383360,
|
| 33 |
+
"active_bytes_peak": 113343831040,
|
| 34 |
+
"allocated_bytes_current": 77298383360,
|
| 35 |
+
"allocated_bytes_peak": 113343831040,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1206404608,
|
| 38 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 115242696704,
|
| 46 |
+
"reserved_bytes_peak": 115242696704,
|
| 47 |
+
"segment_count_current": 1923,
|
| 48 |
+
"segment_count_peak": 1923,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.895755Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1206404608,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115242696704,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 115242696704,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1923,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1923,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:45.240454Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ac7edbe46c6155a1d9a064c5c24884e487feb90b7fc1401f2f098c7baecb476d
|
| 3 |
+
size 1839099
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77315426816,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113351470080,
|
| 5 |
+
"active_bytes_peak_delta": 7639040,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77315426816,
|
| 8 |
+
"active_bytes_peak": 113351470080,
|
| 9 |
+
"allocated_bytes_current": 77315426816,
|
| 10 |
+
"allocated_bytes_peak": 113351470080,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1315190272,
|
| 13 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 116230455296,
|
| 21 |
+
"reserved_bytes_peak": 116230455296,
|
| 22 |
+
"segment_count_current": 1696,
|
| 23 |
+
"segment_count_peak": 1923,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:47.859107Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77315426816,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113351470080,
|
| 30 |
+
"allocated_bytes_peak_delta": 7639040,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77298383360,
|
| 33 |
+
"active_bytes_peak": 113343831040,
|
| 34 |
+
"allocated_bytes_current": 77298383360,
|
| 35 |
+
"allocated_bytes_peak": 113343831040,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1206404608,
|
| 38 |
+
"inactive_split_bytes_peak": 9362196992,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 115242696704,
|
| 46 |
+
"reserved_bytes_peak": 115242696704,
|
| 47 |
+
"segment_count_current": 1923,
|
| 48 |
+
"segment_count_peak": 1923,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:45.244071Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1315190272,
|
| 55 |
+
"inactive_split_bytes_current_delta": 108785664,
|
| 56 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1149253120,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116230455296,
|
| 68 |
+
"reserved_bytes_current_delta": 987758592,
|
| 69 |
+
"reserved_bytes_peak": 116230455296,
|
| 70 |
+
"reserved_bytes_peak_delta": 987758592,
|
| 71 |
+
"segment_count_current": 1696,
|
| 72 |
+
"segment_count_current_delta": -227,
|
| 73 |
+
"segment_count_peak": 1923,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:47.897560Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:189e0599ddd5f751d4966b5bc19cf04a1f32edf44107bc89f91ca9060c9bee95
|
| 3 |
+
size 2181131
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77315426816,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113351470080,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77315426816,
|
| 8 |
+
"active_bytes_peak": 113351470080,
|
| 9 |
+
"allocated_bytes_current": 77315426816,
|
| 10 |
+
"allocated_bytes_peak": 113351470080,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1315190272,
|
| 13 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 116230455296,
|
| 21 |
+
"reserved_bytes_peak": 116230455296,
|
| 22 |
+
"segment_count_current": 1696,
|
| 23 |
+
"segment_count_peak": 1923,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:48.200437Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77315426816,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113351470080,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77315426816,
|
| 33 |
+
"active_bytes_peak": 113351470080,
|
| 34 |
+
"allocated_bytes_current": 77315426816,
|
| 35 |
+
"allocated_bytes_peak": 113351470080,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1315190272,
|
| 38 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 116230455296,
|
| 46 |
+
"reserved_bytes_peak": 116230455296,
|
| 47 |
+
"segment_count_current": 1696,
|
| 48 |
+
"segment_count_peak": 1923,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:47.901776Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1315190272,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116230455296,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 116230455296,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1696,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1923,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:48.209298Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4eecba693a4d72b91d813578a83b637d48f90ddfe5cfcbd277eecd5701aa4ce2
|
| 3 |
+
size 418854
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77315426816,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113351470080,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77315426816,
|
| 8 |
+
"active_bytes_peak": 113351470080,
|
| 9 |
+
"allocated_bytes_current": 77315426816,
|
| 10 |
+
"allocated_bytes_peak": 113351470080,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 1315190272,
|
| 13 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 116230455296,
|
| 21 |
+
"reserved_bytes_peak": 116230455296,
|
| 22 |
+
"segment_count_current": 1696,
|
| 23 |
+
"segment_count_peak": 1923,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:48.509226Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77315426816,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113351470080,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77315426816,
|
| 33 |
+
"active_bytes_peak": 113351470080,
|
| 34 |
+
"allocated_bytes_current": 77315426816,
|
| 35 |
+
"allocated_bytes_peak": 113351470080,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 1315190272,
|
| 38 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 116230455296,
|
| 46 |
+
"reserved_bytes_peak": 116230455296,
|
| 47 |
+
"segment_count_current": 1696,
|
| 48 |
+
"segment_count_peak": 1923,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:48.210418Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1315190272,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 10511450112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116230455296,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 116230455296,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1696,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1923,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:48.518016Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a6184337a698a26e0528082a885307cd77aaee32329aa3ad51f78c8bc74eb1ac
|
| 3 |
+
size 418854
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 32778758656,
|
| 4 |
+
"active_bytes_peak": 82480875008,
|
| 5 |
+
"active_bytes_peak_delta": 37934908928,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 82480875008,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 82480875008,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 90267713536,
|
| 21 |
+
"reserved_bytes_peak": 90267713536,
|
| 22 |
+
"segment_count_current": 1718,
|
| 23 |
+
"segment_count_peak": 1718,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:43.968739Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 32778758656,
|
| 29 |
+
"allocated_bytes_peak": 82480875008,
|
| 30 |
+
"allocated_bytes_peak_delta": 37934908928,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 421167104,
|
| 38 |
+
"inactive_split_bytes_peak": 1025146880,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:17.937156Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 947639808,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 8523717120,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90267713536,
|
| 68 |
+
"reserved_bytes_current_delta": 42016440320,
|
| 69 |
+
"reserved_bytes_peak": 90267713536,
|
| 70 |
+
"reserved_bytes_peak_delta": 42016440320,
|
| 71 |
+
"segment_count_current": 1718,
|
| 72 |
+
"segment_count_current_delta": 1097,
|
| 73 |
+
"segment_count_peak": 1718,
|
| 74 |
+
"segment_count_peak_delta": 1097,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.009111Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bd63bf0418f543d422370e8d8d4fe5ffdc6f159df955028160cd3a2d0c88c29f
|
| 3 |
+
size 2304358
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113344452608,
|
| 5 |
+
"active_bytes_peak_delta": 30863577600,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 113344452608,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 113344452608,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 115364331520,
|
| 21 |
+
"reserved_bytes_peak": 115364331520,
|
| 22 |
+
"segment_count_current": 1934,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:44.855039Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113344452608,
|
| 30 |
+
"allocated_bytes_peak_delta": 30863577600,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 82480875008,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 82480875008,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 90267713536,
|
| 46 |
+
"reserved_bytes_peak": 90267713536,
|
| 47 |
+
"segment_count_current": 1718,
|
| 48 |
+
"segment_count_peak": 1718,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.013716Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115364331520,
|
| 68 |
+
"reserved_bytes_current_delta": 25096617984,
|
| 69 |
+
"reserved_bytes_peak": 115364331520,
|
| 70 |
+
"reserved_bytes_peak_delta": 25096617984,
|
| 71 |
+
"segment_count_current": 1934,
|
| 72 |
+
"segment_count_current_delta": 216,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 216,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.892389Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ecc694e58f210966e555161d3a63de0fdffbbb234ab434109bef0b153934e320
|
| 3 |
+
size 1833396
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113344452608,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 113344452608,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 113344452608,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 115364331520,
|
| 21 |
+
"reserved_bytes_peak": 115364331520,
|
| 22 |
+
"segment_count_current": 1934,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:45.209053Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113344452608,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 113344452608,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 113344452608,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 115364331520,
|
| 46 |
+
"reserved_bytes_peak": 115364331520,
|
| 47 |
+
"segment_count_current": 1934,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.896325Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115364331520,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 115364331520,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1934,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:45.239689Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eb9fe89f6b89e5abf53df921b3b39cce8a047024604744279461e1fd96a10be0
|
| 3 |
+
size 1828450
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 8193024,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:47.858997Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 8193024,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 113344452608,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 113344452608,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 115364331520,
|
| 46 |
+
"reserved_bytes_peak": 115364331520,
|
| 47 |
+
"segment_count_current": 1934,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:45.243253Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 108785664,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1124988416,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 1029701632,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 1029701632,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": -233,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:47.897085Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:45316b56ee65848ab6595cb2c9fe9646bc514f9e4526201f8bd7952224ce759a
|
| 3 |
+
size 2193727
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:48.200462Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77316602368,
|
| 33 |
+
"active_bytes_peak": 113352645632,
|
| 34 |
+
"allocated_bytes_current": 77316602368,
|
| 35 |
+
"allocated_bytes_peak": 113352645632,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1477592576,
|
| 38 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 116394033152,
|
| 46 |
+
"reserved_bytes_peak": 116394033152,
|
| 47 |
+
"segment_count_current": 1701,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:47.901088Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:48.209181Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05272f421f21a224b022b0c8f2193e5cd7d2b5f296dee60b14863343931dce7b
|
| 3 |
+
size 411770
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:48.509218Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77316602368,
|
| 33 |
+
"active_bytes_peak": 113352645632,
|
| 34 |
+
"allocated_bytes_current": 77316602368,
|
| 35 |
+
"allocated_bytes_peak": 113352645632,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1477592576,
|
| 38 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 116394033152,
|
| 46 |
+
"reserved_bytes_peak": 116394033152,
|
| 47 |
+
"segment_count_current": 1701,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:48.210270Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:48.518027Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dc4c76682af535734ff96acf66ab2758651d9119bab6ea704c4c024e0cab9e14
|
| 3 |
+
size 411770
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 32778758656,
|
| 4 |
+
"active_bytes_peak": 82480875008,
|
| 5 |
+
"active_bytes_peak_delta": 37934908928,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 82480875008,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 82480875008,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 90267713536,
|
| 21 |
+
"reserved_bytes_peak": 90267713536,
|
| 22 |
+
"segment_count_current": 1718,
|
| 23 |
+
"segment_count_peak": 1718,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:43.968843Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 32778758656,
|
| 29 |
+
"allocated_bytes_peak": 82480875008,
|
| 30 |
+
"allocated_bytes_peak_delta": 37934908928,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 421167104,
|
| 38 |
+
"inactive_split_bytes_peak": 1025146880,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:17.937342Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 947639808,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 8523717120,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90267713536,
|
| 68 |
+
"reserved_bytes_current_delta": 42016440320,
|
| 69 |
+
"reserved_bytes_peak": 90267713536,
|
| 70 |
+
"reserved_bytes_peak_delta": 42016440320,
|
| 71 |
+
"segment_count_current": 1718,
|
| 72 |
+
"segment_count_current_delta": 1097,
|
| 73 |
+
"segment_count_peak": 1718,
|
| 74 |
+
"segment_count_peak_delta": 1097,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.009608Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:815384b03107533a05df19ec1085b9480eefa54d8bd0488516d73ea922bb3ecd
|
| 3 |
+
size 2304358
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113344452608,
|
| 5 |
+
"active_bytes_peak_delta": 30863577600,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 113344452608,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 113344452608,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 115364331520,
|
| 21 |
+
"reserved_bytes_peak": 115364331520,
|
| 22 |
+
"segment_count_current": 1934,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:44.855177Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113344452608,
|
| 30 |
+
"allocated_bytes_peak_delta": 30863577600,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 82480875008,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 82480875008,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 90267713536,
|
| 46 |
+
"reserved_bytes_peak": 90267713536,
|
| 47 |
+
"segment_count_current": 1718,
|
| 48 |
+
"segment_count_peak": 1718,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.014261Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115364331520,
|
| 68 |
+
"reserved_bytes_current_delta": 25096617984,
|
| 69 |
+
"reserved_bytes_peak": 115364331520,
|
| 70 |
+
"reserved_bytes_peak_delta": 25096617984,
|
| 71 |
+
"segment_count_current": 1934,
|
| 72 |
+
"segment_count_current_delta": 216,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 216,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.892180Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a0e924ed7d72f88dd565acd94a3993cca8869ec8eaf1d717b05d561c9cf5939f
|
| 3 |
+
size 1833399
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113344452608,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 113344452608,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 113344452608,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 115364331520,
|
| 21 |
+
"reserved_bytes_peak": 115364331520,
|
| 22 |
+
"segment_count_current": 1934,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:45.209140Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113344452608,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 113344452608,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 113344452608,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 115364331520,
|
| 46 |
+
"reserved_bytes_peak": 115364331520,
|
| 47 |
+
"segment_count_current": 1934,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.896151Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115364331520,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 115364331520,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1934,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:45.240203Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6d8f6f12368938c04a7a33415f661c3e6e261df0a86be8fe035119b6f60a55ce
|
| 3 |
+
size 1828562
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 8193024,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:47.859142Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 8193024,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 113344452608,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 113344452608,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 115364331520,
|
| 46 |
+
"reserved_bytes_peak": 115364331520,
|
| 47 |
+
"segment_count_current": 1934,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:45.243780Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 108785664,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1124988416,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 1029701632,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 1029701632,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": -233,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:47.897687Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:be9b4543ec95a2ba876ec647c3bf34f42c3c7d64e3aa58f3bc5c440b6f7afeb3
|
| 3 |
+
size 2193889
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:48.200546Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77316602368,
|
| 33 |
+
"active_bytes_peak": 113352645632,
|
| 34 |
+
"allocated_bytes_current": 77316602368,
|
| 35 |
+
"allocated_bytes_peak": 113352645632,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1477592576,
|
| 38 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 116394033152,
|
| 46 |
+
"reserved_bytes_peak": 116394033152,
|
| 47 |
+
"segment_count_current": 1701,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:47.902545Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:48.209332Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b3a4d3b1d51744d2bb80d93fbaf542d0cd7ef040974de68a67e6b149c6e0dcdc
|
| 3 |
+
size 411608
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:48.509293Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77316602368,
|
| 33 |
+
"active_bytes_peak": 113352645632,
|
| 34 |
+
"allocated_bytes_current": 77316602368,
|
| 35 |
+
"allocated_bytes_peak": 113352645632,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1477592576,
|
| 38 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 116394033152,
|
| 46 |
+
"reserved_bytes_peak": 116394033152,
|
| 47 |
+
"segment_count_current": 1701,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:48.210429Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:48.518004Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:964457ab0632888b1b0e58ff3c1f51b88f659ffb6b3f8e3cbdb133b3f8d503ad
|
| 3 |
+
size 411608
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 32778758656,
|
| 4 |
+
"active_bytes_peak": 82480875008,
|
| 5 |
+
"active_bytes_peak_delta": 37934908928,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 82480875008,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 82480875008,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 90267713536,
|
| 21 |
+
"reserved_bytes_peak": 90267713536,
|
| 22 |
+
"segment_count_current": 1718,
|
| 23 |
+
"segment_count_peak": 1718,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:43.968838Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 32778758656,
|
| 29 |
+
"allocated_bytes_peak": 82480875008,
|
| 30 |
+
"allocated_bytes_peak_delta": 37934908928,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 421167104,
|
| 38 |
+
"inactive_split_bytes_peak": 1025146880,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:17.937038Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 947639808,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 8523717120,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90267713536,
|
| 68 |
+
"reserved_bytes_current_delta": 42016440320,
|
| 69 |
+
"reserved_bytes_peak": 90267713536,
|
| 70 |
+
"reserved_bytes_peak_delta": 42016440320,
|
| 71 |
+
"segment_count_current": 1718,
|
| 72 |
+
"segment_count_current_delta": 1097,
|
| 73 |
+
"segment_count_peak": 1718,
|
| 74 |
+
"segment_count_peak_delta": 1097,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.008245Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1f8d9f6081984ccd5c5fe51aab0a3e3e8467fdc958903e68857b52025e995c65
|
| 3 |
+
size 2304358
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113344452608,
|
| 5 |
+
"active_bytes_peak_delta": 30863577600,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 113344452608,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 113344452608,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 115364331520,
|
| 21 |
+
"reserved_bytes_peak": 115364331520,
|
| 22 |
+
"segment_count_current": 1934,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:44.855203Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113344452608,
|
| 30 |
+
"allocated_bytes_peak_delta": 30863577600,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 82480875008,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 82480875008,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 90267713536,
|
| 46 |
+
"reserved_bytes_peak": 90267713536,
|
| 47 |
+
"segment_count_current": 1718,
|
| 48 |
+
"segment_count_peak": 1718,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.012798Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115364331520,
|
| 68 |
+
"reserved_bytes_current_delta": 25096617984,
|
| 69 |
+
"reserved_bytes_peak": 115364331520,
|
| 70 |
+
"reserved_bytes_peak_delta": 25096617984,
|
| 71 |
+
"segment_count_current": 1934,
|
| 72 |
+
"segment_count_current_delta": 216,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 216,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:44.892124Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e16105fc8a481ebf77593be97dde94d82e8367120bc404653621454fbc513eeb
|
| 3 |
+
size 1833396
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77299558912,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113344452608,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77299558912,
|
| 8 |
+
"active_bytes_peak": 113344452608,
|
| 9 |
+
"allocated_bytes_current": 77299558912,
|
| 10 |
+
"allocated_bytes_peak": 113344452608,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1368806912,
|
| 13 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 115364331520,
|
| 21 |
+
"reserved_bytes_peak": 115364331520,
|
| 22 |
+
"segment_count_current": 1934,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:45.209139Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77299558912,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113344452608,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 113344452608,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 113344452608,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 115364331520,
|
| 46 |
+
"reserved_bytes_peak": 115364331520,
|
| 47 |
+
"segment_count_current": 1934,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:44.896027Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1368806912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 115364331520,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 115364331520,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1934,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:45.240436Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:04cad6d6b0d89aa3a18cdbe7ae40d41a1b82d05deda56a0d48972a25f5405133
|
| 3 |
+
size 1828450
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 8193024,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:47.859143Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 8193024,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77299558912,
|
| 33 |
+
"active_bytes_peak": 113344452608,
|
| 34 |
+
"allocated_bytes_current": 77299558912,
|
| 35 |
+
"allocated_bytes_peak": 113344452608,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1368806912,
|
| 38 |
+
"inactive_split_bytes_peak": 9548864000,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 115364331520,
|
| 46 |
+
"reserved_bytes_peak": 115364331520,
|
| 47 |
+
"segment_count_current": 1934,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:45.243981Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 108785664,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1124988416,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 1029701632,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 1029701632,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": -233,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:47.897322Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2e52652eb4021b76676884a199a82ac80943346346a366f19bfbcad71fabb5b7
|
| 3 |
+
size 2193725
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77316602368,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113352645632,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77316602368,
|
| 8 |
+
"active_bytes_peak": 113352645632,
|
| 9 |
+
"allocated_bytes_current": 77316602368,
|
| 10 |
+
"allocated_bytes_peak": 113352645632,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1477592576,
|
| 13 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 116394033152,
|
| 21 |
+
"reserved_bytes_peak": 116394033152,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1934,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:11:48.200547Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77316602368,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113352645632,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77316602368,
|
| 33 |
+
"active_bytes_peak": 113352645632,
|
| 34 |
+
"allocated_bytes_current": 77316602368,
|
| 35 |
+
"allocated_bytes_peak": 113352645632,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1477592576,
|
| 38 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 116394033152,
|
| 46 |
+
"reserved_bytes_peak": 116394033152,
|
| 47 |
+
"segment_count_current": 1701,
|
| 48 |
+
"segment_count_peak": 1934,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:11:47.901458Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1477592576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 10673852416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 116394033152,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 116394033152,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1934,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:11:48.209259Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fe460f29325376f03ccea0720aacafe5fe7c016cd04f88b0a95e2395321f6b41
|
| 3 |
+
size 411768
|