granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 4)
Browse files- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env +61 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log +0 -0
- ultraep/tierbmem/tierbmem_gbs8/results.tsv +8 -0
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 76000535552,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 112732114432,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 76000535552,
|
| 8 |
+
"active_bytes_peak": 112732114432,
|
| 9 |
+
"allocated_bytes_current": 76000535552,
|
| 10 |
+
"allocated_bytes_peak": 112732114432,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 4091796480,
|
| 13 |
+
"inactive_split_bytes_peak": 15975386112,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 113170710528,
|
| 21 |
+
"reserved_bytes_peak": 113170710528,
|
| 22 |
+
"segment_count_current": 2765,
|
| 23 |
+
"segment_count_peak": 2765,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:20:11.781814Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 76000535552,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 112732114432,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 76000535552,
|
| 33 |
+
"active_bytes_peak": 112732114432,
|
| 34 |
+
"allocated_bytes_current": 76000535552,
|
| 35 |
+
"allocated_bytes_peak": 112732114432,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 4091796480,
|
| 38 |
+
"inactive_split_bytes_peak": 14970116096,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 113170710528,
|
| 46 |
+
"reserved_bytes_peak": 113170710528,
|
| 47 |
+
"segment_count_current": 2765,
|
| 48 |
+
"segment_count_peak": 2765,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:20:10.856506Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4091796480,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15975386112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1005270016,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113170710528,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113170710528,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2765,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2765,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:20:11.906915Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f9838581342276b9c162cf4a7b7546397f8959dd8f3ba6181b675bd5c6c8aaa0
|
| 3 |
+
size 7089973
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 76000535552,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 112732114432,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 76000535552,
|
| 8 |
+
"active_bytes_peak": 112732114432,
|
| 9 |
+
"allocated_bytes_current": 76000535552,
|
| 10 |
+
"allocated_bytes_peak": 112732114432,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 4091796480,
|
| 13 |
+
"inactive_split_bytes_peak": 15975386112,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 113170710528,
|
| 21 |
+
"reserved_bytes_peak": 113170710528,
|
| 22 |
+
"segment_count_current": 2765,
|
| 23 |
+
"segment_count_peak": 2765,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:20:12.835325Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 76000535552,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 112732114432,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 76000535552,
|
| 33 |
+
"active_bytes_peak": 112732114432,
|
| 34 |
+
"allocated_bytes_current": 76000535552,
|
| 35 |
+
"allocated_bytes_peak": 112732114432,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 4091796480,
|
| 38 |
+
"inactive_split_bytes_peak": 15975386112,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 113170710528,
|
| 46 |
+
"reserved_bytes_peak": 113170710528,
|
| 47 |
+
"segment_count_current": 2765,
|
| 48 |
+
"segment_count_peak": 2765,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:20:11.922365Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4091796480,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15975386112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113170710528,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113170710528,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2765,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2765,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:20:12.966943Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:757b50ebefe28ced1e1146b679a1ff53b36e1fd81159288cd99c4525ec40d465
|
| 3 |
+
size 7089530
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75985415168,
|
| 3 |
+
"active_bytes_current_delta": 32431245312,
|
| 4 |
+
"active_bytes_peak": 81892606976,
|
| 5 |
+
"active_bytes_peak_delta": 38332145664,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75985415168,
|
| 8 |
+
"active_bytes_peak": 81892606976,
|
| 9 |
+
"allocated_bytes_current": 75985415168,
|
| 10 |
+
"allocated_bytes_peak": 81892606976,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4234843136,
|
| 13 |
+
"inactive_split_bytes_peak": 13245673472,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 82317410304,
|
| 21 |
+
"reserved_bytes_peak": 82317410304,
|
| 22 |
+
"segment_count_current": 2042,
|
| 23 |
+
"segment_count_peak": 2042,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:20:07.567056Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75985415168,
|
| 28 |
+
"allocated_bytes_current_delta": 32431245312,
|
| 29 |
+
"allocated_bytes_peak": 81892606976,
|
| 30 |
+
"allocated_bytes_peak_delta": 38332145664,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43554169856,
|
| 33 |
+
"active_bytes_peak": 43560461312,
|
| 34 |
+
"allocated_bytes_current": 43554169856,
|
| 35 |
+
"allocated_bytes_peak": 43560461312,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1255676928,
|
| 38 |
+
"inactive_split_bytes_peak": 1612285952,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 44835012608,
|
| 46 |
+
"reserved_bytes_peak": 44835012608,
|
| 47 |
+
"segment_count_current": 883,
|
| 48 |
+
"segment_count_peak": 883,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:19:39.070373Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4234843136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2979166208,
|
| 56 |
+
"inactive_split_bytes_peak": 13245673472,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 11633387520,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 82317410304,
|
| 68 |
+
"reserved_bytes_current_delta": 37482397696,
|
| 69 |
+
"reserved_bytes_peak": 82317410304,
|
| 70 |
+
"reserved_bytes_peak_delta": 37482397696,
|
| 71 |
+
"segment_count_current": 2042,
|
| 72 |
+
"segment_count_current_delta": 1159,
|
| 73 |
+
"segment_count_peak": 2042,
|
| 74 |
+
"segment_count_peak_delta": 1159,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:20:07.703056Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:56d4a3bf31c687a59019c181fe99c388241d50779613a349e4d9a40b6d9854b8
|
| 3 |
+
size 6879462
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75985415168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 112694591488,
|
| 5 |
+
"active_bytes_peak_delta": 30801984512,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75985415168,
|
| 8 |
+
"active_bytes_peak": 112694591488,
|
| 9 |
+
"allocated_bytes_current": 75985415168,
|
| 10 |
+
"allocated_bytes_peak": 112694591488,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4234843136,
|
| 13 |
+
"inactive_split_bytes_peak": 13245673472,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 113179099136,
|
| 21 |
+
"reserved_bytes_peak": 113179099136,
|
| 22 |
+
"segment_count_current": 2733,
|
| 23 |
+
"segment_count_peak": 2733,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:20:09.539414Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75985415168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 112694591488,
|
| 30 |
+
"allocated_bytes_peak_delta": 30801984512,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75985415168,
|
| 33 |
+
"active_bytes_peak": 81892606976,
|
| 34 |
+
"allocated_bytes_current": 75985415168,
|
| 35 |
+
"allocated_bytes_peak": 81892606976,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4234843136,
|
| 38 |
+
"inactive_split_bytes_peak": 13245673472,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 82317410304,
|
| 46 |
+
"reserved_bytes_peak": 82317410304,
|
| 47 |
+
"segment_count_current": 2042,
|
| 48 |
+
"segment_count_peak": 2042,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:20:07.723737Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4234843136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13245673472,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113179099136,
|
| 68 |
+
"reserved_bytes_current_delta": 30861688832,
|
| 69 |
+
"reserved_bytes_peak": 113179099136,
|
| 70 |
+
"reserved_bytes_peak_delta": 30861688832,
|
| 71 |
+
"segment_count_current": 2733,
|
| 72 |
+
"segment_count_current_delta": 691,
|
| 73 |
+
"segment_count_peak": 2733,
|
| 74 |
+
"segment_count_peak_delta": 691,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:20:09.675108Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:367a06fba3055ac810e3611c1bec6ce9ebc804b39eed83a348e25f043a9f2839
|
| 3 |
+
size 7113304
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75985415168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 112694591488,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75985415168,
|
| 8 |
+
"active_bytes_peak": 112694591488,
|
| 9 |
+
"allocated_bytes_current": 75985415168,
|
| 10 |
+
"allocated_bytes_peak": 112694591488,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4234843136,
|
| 13 |
+
"inactive_split_bytes_peak": 14533562368,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 113179099136,
|
| 21 |
+
"reserved_bytes_peak": 113179099136,
|
| 22 |
+
"segment_count_current": 2733,
|
| 23 |
+
"segment_count_peak": 2733,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:20:10.713592Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75985415168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 112694591488,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75985415168,
|
| 33 |
+
"active_bytes_peak": 112694591488,
|
| 34 |
+
"allocated_bytes_current": 75985415168,
|
| 35 |
+
"allocated_bytes_peak": 112694591488,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4234843136,
|
| 38 |
+
"inactive_split_bytes_peak": 13245673472,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 113179099136,
|
| 46 |
+
"reserved_bytes_peak": 113179099136,
|
| 47 |
+
"segment_count_current": 2733,
|
| 48 |
+
"segment_count_peak": 2733,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:20:09.691456Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4234843136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14533562368,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1287888896,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113179099136,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113179099136,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2733,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2733,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:20:10.845013Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5f4f68264259ae7b5ea43c2dd7b7662c48fc98f26ba615d185626ae0913a0884
|
| 3 |
+
size 7074711
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75985415168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 112694591488,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75985415168,
|
| 8 |
+
"active_bytes_peak": 112694591488,
|
| 9 |
+
"allocated_bytes_current": 75985415168,
|
| 10 |
+
"allocated_bytes_peak": 112694591488,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4234843136,
|
| 13 |
+
"inactive_split_bytes_peak": 14899931136,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 113179099136,
|
| 21 |
+
"reserved_bytes_peak": 113179099136,
|
| 22 |
+
"segment_count_current": 2733,
|
| 23 |
+
"segment_count_peak": 2733,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:20:11.782011Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75985415168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 112694591488,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75985415168,
|
| 33 |
+
"active_bytes_peak": 112694591488,
|
| 34 |
+
"allocated_bytes_current": 75985415168,
|
| 35 |
+
"allocated_bytes_peak": 112694591488,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4234843136,
|
| 38 |
+
"inactive_split_bytes_peak": 14533562368,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 113179099136,
|
| 46 |
+
"reserved_bytes_peak": 113179099136,
|
| 47 |
+
"segment_count_current": 2733,
|
| 48 |
+
"segment_count_peak": 2733,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:20:10.860754Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4234843136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14899931136,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 366368768,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113179099136,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113179099136,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2733,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2733,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:20:11.914163Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f76952e21e735d7c7699ecdbdc0db523d6e05c2fe04e373d5f20bde68b07089c
|
| 3 |
+
size 7073460
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75985415168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 112694591488,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75985415168,
|
| 8 |
+
"active_bytes_peak": 112694591488,
|
| 9 |
+
"allocated_bytes_current": 75985415168,
|
| 10 |
+
"allocated_bytes_peak": 112694591488,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4234843136,
|
| 13 |
+
"inactive_split_bytes_peak": 15641051136,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 113179099136,
|
| 21 |
+
"reserved_bytes_peak": 113179099136,
|
| 22 |
+
"segment_count_current": 2733,
|
| 23 |
+
"segment_count_peak": 2733,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:20:12.835501Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75985415168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 112694591488,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75985415168,
|
| 33 |
+
"active_bytes_peak": 112694591488,
|
| 34 |
+
"allocated_bytes_current": 75985415168,
|
| 35 |
+
"allocated_bytes_peak": 112694591488,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4234843136,
|
| 38 |
+
"inactive_split_bytes_peak": 14899931136,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 113179099136,
|
| 46 |
+
"reserved_bytes_peak": 113179099136,
|
| 47 |
+
"segment_count_current": 2733,
|
| 48 |
+
"segment_count_peak": 2733,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:20:11.930240Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4234843136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15641051136,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 741120000,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113179099136,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113179099136,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2733,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2733,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:20:12.964841Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2406225f221a5445829fd208699e27b38ea629b0de6006b846b2c4a78180e00d
|
| 3 |
+
size 7073201
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=QWEN3_30B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-better-memory-dump-support
|
| 6 |
+
commit=c3a8a8a7
|
| 7 |
+
num_layers=48
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=6144
|
| 10 |
+
moe_ffn_hidden=768
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=4718592
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=48
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=0
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=0
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=4096
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=32768
|
| 39 |
+
mean_tokens_per_expert=2048
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=16
|
| 42 |
+
capacity_mode=dynamic
|
| 43 |
+
capacity=dynamic
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=0
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=memory
|
| 48 |
+
memory_dump_mode=iteration
|
| 49 |
+
memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=5
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=256
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
ultraep/tierbmem/tierbmem_gbs8/results.tsv
CHANGED
|
@@ -1 +1,9 @@
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 2 |
+
QWEN3_235B ultraep matched 16 8192 8 1.00 none 1 OK 308.70 308.70 121321 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1
|
| 3 |
+
QWEN3_235B ultraep matched 16 8192 8 1.125 none 1 OK 310.90 310.90 122445 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1
|
| 4 |
+
QWEN3_235B ultraep matched 16 8192 8 1.25 none 1 OK 312.40 312.40 123775 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1
|
| 5 |
+
QWEN3_235B ultraep matched 16 8192 8 dynamic none 1 OK NA NA 120703 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1
|
| 6 |
+
QWEN3_30B ultraep matched 16 4096 8 1.00 none 1 OK 375.70 375.70 118839 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1
|
| 7 |
+
QWEN3_30B ultraep matched 16 4096 8 1.125 none 1 OK 381.00 381.00 121803 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1
|
| 8 |
+
QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 382.90 382.90 124053 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1
|
| 9 |
+
QWEN3_30B ultraep matched 16 4096 8 dynamic none 1 OK NA NA 116131 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1
|