granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 3)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle +3 -0
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75901254656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113456552448,
|
| 5 |
+
"active_bytes_peak_delta": 30836036096,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75901254656,
|
| 8 |
+
"active_bytes_peak": 113456552448,
|
| 9 |
+
"allocated_bytes_current": 75901254656,
|
| 10 |
+
"allocated_bytes_peak": 113456552448,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 5912839168,
|
| 13 |
+
"inactive_split_bytes_peak": 14139202560,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 113873256448,
|
| 21 |
+
"reserved_bytes_peak": 113873256448,
|
| 22 |
+
"segment_count_current": 2836,
|
| 23 |
+
"segment_count_peak": 2836,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:37.348676Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75901254656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113456552448,
|
| 30 |
+
"allocated_bytes_peak_delta": 30836036096,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75901254656,
|
| 33 |
+
"active_bytes_peak": 82620516352,
|
| 34 |
+
"allocated_bytes_current": 75901254656,
|
| 35 |
+
"allocated_bytes_peak": 82620516352,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 5912839168,
|
| 38 |
+
"inactive_split_bytes_peak": 14139202560,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 83022053376,
|
| 46 |
+
"reserved_bytes_peak": 83022053376,
|
| 47 |
+
"segment_count_current": 2125,
|
| 48 |
+
"segment_count_peak": 2125,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:36.067939Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5912839168,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14139202560,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113873256448,
|
| 68 |
+
"reserved_bytes_current_delta": 30851203072,
|
| 69 |
+
"reserved_bytes_peak": 113873256448,
|
| 70 |
+
"reserved_bytes_peak_delta": 30851203072,
|
| 71 |
+
"segment_count_current": 2836,
|
| 72 |
+
"segment_count_current_delta": 711,
|
| 73 |
+
"segment_count_peak": 2836,
|
| 74 |
+
"segment_count_peak_delta": 711,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:37.476498Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cd22d0b31afd0255b04015d9a7b4e9df333063e40f147439b23bec488ad4cdcf
|
| 3 |
+
size 7113911
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75901254656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113495366144,
|
| 5 |
+
"active_bytes_peak_delta": 38813696,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75901254656,
|
| 8 |
+
"active_bytes_peak": 113495366144,
|
| 9 |
+
"allocated_bytes_current": 75901254656,
|
| 10 |
+
"allocated_bytes_peak": 113495366144,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 5912839168,
|
| 13 |
+
"inactive_split_bytes_peak": 14139202560,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 113873256448,
|
| 21 |
+
"reserved_bytes_peak": 113873256448,
|
| 22 |
+
"segment_count_current": 2836,
|
| 23 |
+
"segment_count_peak": 2836,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:38.517239Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75901254656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113495366144,
|
| 30 |
+
"allocated_bytes_peak_delta": 38813696,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75901254656,
|
| 33 |
+
"active_bytes_peak": 113456552448,
|
| 34 |
+
"allocated_bytes_current": 75901254656,
|
| 35 |
+
"allocated_bytes_peak": 113456552448,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 5912839168,
|
| 38 |
+
"inactive_split_bytes_peak": 14139202560,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 113873256448,
|
| 46 |
+
"reserved_bytes_peak": 113873256448,
|
| 47 |
+
"segment_count_current": 2836,
|
| 48 |
+
"segment_count_peak": 2836,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:37.492292Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5912839168,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14139202560,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113873256448,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113873256448,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2836,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2836,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:38.647254Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:982257ddd1e947c99f2570403f65f61ca2f011471cd1462fb9c5af7cffa8c375
|
| 3 |
+
size 6892313
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75918298112,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113495366144,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75918298112,
|
| 8 |
+
"active_bytes_peak": 113495366144,
|
| 9 |
+
"allocated_bytes_current": 75918298112,
|
| 10 |
+
"allocated_bytes_peak": 113495366144,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 5950321664,
|
| 13 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 119042736128,
|
| 21 |
+
"reserved_bytes_peak": 119042736128,
|
| 22 |
+
"segment_count_current": 3126,
|
| 23 |
+
"segment_count_peak": 3126,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:41.821331Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75918298112,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113495366144,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75901254656,
|
| 33 |
+
"active_bytes_peak": 113495366144,
|
| 34 |
+
"allocated_bytes_current": 75901254656,
|
| 35 |
+
"allocated_bytes_peak": 113495366144,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 5912839168,
|
| 38 |
+
"inactive_split_bytes_peak": 14139202560,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 113873256448,
|
| 46 |
+
"reserved_bytes_peak": 113873256448,
|
| 47 |
+
"segment_count_current": 2836,
|
| 48 |
+
"segment_count_peak": 2836,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:38.662900Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5950321664,
|
| 55 |
+
"inactive_split_bytes_current_delta": 37482496,
|
| 56 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1328300544,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119042736128,
|
| 68 |
+
"reserved_bytes_current_delta": 5169479680,
|
| 69 |
+
"reserved_bytes_peak": 119042736128,
|
| 70 |
+
"reserved_bytes_peak_delta": 5169479680,
|
| 71 |
+
"segment_count_current": 3126,
|
| 72 |
+
"segment_count_current_delta": 290,
|
| 73 |
+
"segment_count_peak": 3126,
|
| 74 |
+
"segment_count_peak_delta": 290,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:41.960602Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7488c72f8849c7acc6825f47d531d1216b71cfd482bf98b9a9258ad406183644
|
| 3 |
+
size 7508905
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75918298112,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113495366144,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75918298112,
|
| 8 |
+
"active_bytes_peak": 113495366144,
|
| 9 |
+
"allocated_bytes_current": 75918298112,
|
| 10 |
+
"allocated_bytes_peak": 113495366144,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 5950321664,
|
| 13 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 119042736128,
|
| 21 |
+
"reserved_bytes_peak": 119042736128,
|
| 22 |
+
"segment_count_current": 3126,
|
| 23 |
+
"segment_count_peak": 3126,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.351495Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75918298112,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113495366144,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75918298112,
|
| 33 |
+
"active_bytes_peak": 113495366144,
|
| 34 |
+
"allocated_bytes_current": 75918298112,
|
| 35 |
+
"allocated_bytes_peak": 113495366144,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 5950321664,
|
| 38 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 119042736128,
|
| 46 |
+
"reserved_bytes_peak": 119042736128,
|
| 47 |
+
"segment_count_current": 3126,
|
| 48 |
+
"segment_count_peak": 3126,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:41.977258Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5950321664,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119042736128,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119042736128,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3126,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3126,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.371970Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:92651b2e399d31cbb87b64130deeff9d01389b8e0872afd4185c48f5b885c6f6
|
| 3 |
+
size 902755
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75918298112,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113495366144,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75918298112,
|
| 8 |
+
"active_bytes_peak": 113495366144,
|
| 9 |
+
"allocated_bytes_current": 75918298112,
|
| 10 |
+
"allocated_bytes_peak": 113495366144,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 5950321664,
|
| 13 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 119042736128,
|
| 21 |
+
"reserved_bytes_peak": 119042736128,
|
| 22 |
+
"segment_count_current": 3126,
|
| 23 |
+
"segment_count_peak": 3126,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.732476Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75918298112,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113495366144,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75918298112,
|
| 33 |
+
"active_bytes_peak": 113495366144,
|
| 34 |
+
"allocated_bytes_current": 75918298112,
|
| 35 |
+
"allocated_bytes_peak": 113495366144,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 5950321664,
|
| 38 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 119042736128,
|
| 46 |
+
"reserved_bytes_peak": 119042736128,
|
| 47 |
+
"segment_count_current": 3126,
|
| 48 |
+
"segment_count_peak": 3126,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:42.373557Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5950321664,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15467503104,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119042736128,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119042736128,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3126,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3126,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.751054Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:87358302e71f8050e4d1f4e0c8b125bcd8f2e0539bc811ebd33be2336b197cb5
|
| 3 |
+
size 902755
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75883125760,
|
| 3 |
+
"active_bytes_current_delta": 32328931328,
|
| 4 |
+
"active_bytes_peak": 82624266240,
|
| 5 |
+
"active_bytes_peak_delta": 39063780352,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75883125760,
|
| 8 |
+
"active_bytes_peak": 82624266240,
|
| 9 |
+
"allocated_bytes_current": 75883125760,
|
| 10 |
+
"allocated_bytes_peak": 82624266240,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 5901607936,
|
| 13 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 82996887552,
|
| 21 |
+
"reserved_bytes_peak": 82996887552,
|
| 22 |
+
"segment_count_current": 2112,
|
| 23 |
+
"segment_count_peak": 2112,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:35.910227Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75883125760,
|
| 28 |
+
"allocated_bytes_current_delta": 32328931328,
|
| 29 |
+
"allocated_bytes_peak": 82624266240,
|
| 30 |
+
"allocated_bytes_peak_delta": 39063780352,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43554194432,
|
| 33 |
+
"active_bytes_peak": 43560485888,
|
| 34 |
+
"allocated_bytes_current": 43554194432,
|
| 35 |
+
"allocated_bytes_peak": 43560485888,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1255652352,
|
| 38 |
+
"inactive_split_bytes_peak": 1612261376,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 44835012608,
|
| 46 |
+
"reserved_bytes_peak": 44835012608,
|
| 47 |
+
"segment_count_current": 883,
|
| 48 |
+
"segment_count_peak": 883,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:09.289391Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5901607936,
|
| 55 |
+
"inactive_split_bytes_current_delta": 4645955584,
|
| 56 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 12537032192,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 82996887552,
|
| 68 |
+
"reserved_bytes_current_delta": 38161874944,
|
| 69 |
+
"reserved_bytes_peak": 82996887552,
|
| 70 |
+
"reserved_bytes_peak_delta": 38161874944,
|
| 71 |
+
"segment_count_current": 2112,
|
| 72 |
+
"segment_count_current_delta": 1229,
|
| 73 |
+
"segment_count_peak": 2112,
|
| 74 |
+
"segment_count_peak_delta": 1229,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:36.047916Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:feb161bcba0cd15af96cbddd476e5bbbbcc7702ae29e002495a161d94622b2b6
|
| 3 |
+
size 7001386
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75883125760,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113455617536,
|
| 5 |
+
"active_bytes_peak_delta": 30831351296,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75883125760,
|
| 8 |
+
"active_bytes_peak": 113455617536,
|
| 9 |
+
"allocated_bytes_current": 75883125760,
|
| 10 |
+
"allocated_bytes_peak": 113455617536,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 5901607936,
|
| 13 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 113841799168,
|
| 21 |
+
"reserved_bytes_peak": 113841799168,
|
| 22 |
+
"segment_count_current": 2812,
|
| 23 |
+
"segment_count_peak": 2812,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:37.348673Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75883125760,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113455617536,
|
| 30 |
+
"allocated_bytes_peak_delta": 30831351296,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75883125760,
|
| 33 |
+
"active_bytes_peak": 82624266240,
|
| 34 |
+
"allocated_bytes_current": 75883125760,
|
| 35 |
+
"allocated_bytes_peak": 82624266240,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 5901607936,
|
| 38 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 82996887552,
|
| 46 |
+
"reserved_bytes_peak": 82996887552,
|
| 47 |
+
"segment_count_current": 2112,
|
| 48 |
+
"segment_count_peak": 2112,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:36.069015Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5901607936,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113841799168,
|
| 68 |
+
"reserved_bytes_current_delta": 30844911616,
|
| 69 |
+
"reserved_bytes_peak": 113841799168,
|
| 70 |
+
"reserved_bytes_peak_delta": 30844911616,
|
| 71 |
+
"segment_count_current": 2812,
|
| 72 |
+
"segment_count_current_delta": 700,
|
| 73 |
+
"segment_count_peak": 2812,
|
| 74 |
+
"segment_count_peak_delta": 700,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:37.477367Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bb407cb73c7f6b58ae60b7500ac00821299baa5e1cabe4319498795df9c8c375
|
| 3 |
+
size 6492477
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75883125760,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113498658304,
|
| 5 |
+
"active_bytes_peak_delta": 43040768,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75883125760,
|
| 8 |
+
"active_bytes_peak": 113498658304,
|
| 9 |
+
"allocated_bytes_current": 75883125760,
|
| 10 |
+
"allocated_bytes_peak": 113498658304,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 5901607936,
|
| 13 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 113841799168,
|
| 21 |
+
"reserved_bytes_peak": 113841799168,
|
| 22 |
+
"segment_count_current": 2812,
|
| 23 |
+
"segment_count_peak": 2812,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:38.517252Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75883125760,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113498658304,
|
| 30 |
+
"allocated_bytes_peak_delta": 43040768,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75883125760,
|
| 33 |
+
"active_bytes_peak": 113455617536,
|
| 34 |
+
"allocated_bytes_current": 75883125760,
|
| 35 |
+
"allocated_bytes_peak": 113455617536,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 5901607936,
|
| 38 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 113841799168,
|
| 46 |
+
"reserved_bytes_peak": 113841799168,
|
| 47 |
+
"segment_count_current": 2812,
|
| 48 |
+
"segment_count_peak": 2812,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:37.492866Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5901607936,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113841799168,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113841799168,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2812,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2812,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:38.646112Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f0b8e83304e96a7c1cdebf66dfd35177382faa8f65936acf9ca3f1ab51cd3c30
|
| 3 |
+
size 6457970
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75900169216,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113498658304,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75900169216,
|
| 8 |
+
"active_bytes_peak": 113498658304,
|
| 9 |
+
"allocated_bytes_current": 75900169216,
|
| 10 |
+
"allocated_bytes_peak": 113498658304,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 5939090432,
|
| 13 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 119013376000,
|
| 21 |
+
"reserved_bytes_peak": 119013376000,
|
| 22 |
+
"segment_count_current": 3111,
|
| 23 |
+
"segment_count_peak": 3111,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:41.821366Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75900169216,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113498658304,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75883125760,
|
| 33 |
+
"active_bytes_peak": 113498658304,
|
| 34 |
+
"allocated_bytes_current": 75883125760,
|
| 35 |
+
"allocated_bytes_peak": 113498658304,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 5901607936,
|
| 38 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 113841799168,
|
| 46 |
+
"reserved_bytes_peak": 113841799168,
|
| 47 |
+
"segment_count_current": 2812,
|
| 48 |
+
"segment_count_peak": 2812,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:38.661216Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5939090432,
|
| 55 |
+
"inactive_split_bytes_current_delta": 37482496,
|
| 56 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1306978304,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119013376000,
|
| 68 |
+
"reserved_bytes_current_delta": 5171576832,
|
| 69 |
+
"reserved_bytes_peak": 119013376000,
|
| 70 |
+
"reserved_bytes_peak_delta": 5171576832,
|
| 71 |
+
"segment_count_current": 3111,
|
| 72 |
+
"segment_count_current_delta": 299,
|
| 73 |
+
"segment_count_peak": 3111,
|
| 74 |
+
"segment_count_peak_delta": 299,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:41.961683Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:10ba6b6f670b2f05f5b3a2ba8a6b103fc236a236a656023fe21288c42e3b6458
|
| 3 |
+
size 7491891
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75900169216,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113498658304,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75900169216,
|
| 8 |
+
"active_bytes_peak": 113498658304,
|
| 9 |
+
"allocated_bytes_current": 75900169216,
|
| 10 |
+
"allocated_bytes_peak": 113498658304,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 5939090432,
|
| 13 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 119013376000,
|
| 21 |
+
"reserved_bytes_peak": 119013376000,
|
| 22 |
+
"segment_count_current": 3111,
|
| 23 |
+
"segment_count_peak": 3111,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.351498Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75900169216,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113498658304,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75900169216,
|
| 33 |
+
"active_bytes_peak": 113498658304,
|
| 34 |
+
"allocated_bytes_current": 75900169216,
|
| 35 |
+
"allocated_bytes_peak": 113498658304,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 5939090432,
|
| 38 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 119013376000,
|
| 46 |
+
"reserved_bytes_peak": 119013376000,
|
| 47 |
+
"segment_count_current": 3111,
|
| 48 |
+
"segment_count_peak": 3111,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:41.978110Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5939090432,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119013376000,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119013376000,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3111,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3111,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.372147Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e4e61955093733498e6a3566b7a8c688e2dc8fe535215b82c6c6ecefde0af079
|
| 3 |
+
size 886237
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75900169216,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113498658304,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75900169216,
|
| 8 |
+
"active_bytes_peak": 113498658304,
|
| 9 |
+
"allocated_bytes_current": 75900169216,
|
| 10 |
+
"allocated_bytes_peak": 113498658304,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 5939090432,
|
| 13 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 119013376000,
|
| 21 |
+
"reserved_bytes_peak": 119013376000,
|
| 22 |
+
"segment_count_current": 3111,
|
| 23 |
+
"segment_count_peak": 3111,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.732501Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75900169216,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113498658304,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75900169216,
|
| 33 |
+
"active_bytes_peak": 113498658304,
|
| 34 |
+
"allocated_bytes_current": 75900169216,
|
| 35 |
+
"allocated_bytes_peak": 113498658304,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 5939090432,
|
| 38 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 119013376000,
|
| 46 |
+
"reserved_bytes_peak": 119013376000,
|
| 47 |
+
"segment_count_current": 3111,
|
| 48 |
+
"segment_count_peak": 3111,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:42.373743Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5939090432,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15456271872,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119013376000,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119013376000,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3111,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3111,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.751076Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05dc5dc4571eeeccdfe5f49c6101cb5aa84f61118e3598b3855f030e6e837266
|
| 3 |
+
size 886237
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75906373632,
|
| 3 |
+
"active_bytes_current_delta": 32348899840,
|
| 4 |
+
"active_bytes_peak": 82622775808,
|
| 5 |
+
"active_bytes_peak_delta": 39059010560,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75906373632,
|
| 8 |
+
"active_bytes_peak": 82622775808,
|
| 9 |
+
"allocated_bytes_current": 75906373632,
|
| 10 |
+
"allocated_bytes_peak": 82622775808,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 5924497408,
|
| 13 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 83022053376,
|
| 21 |
+
"reserved_bytes_peak": 83022053376,
|
| 22 |
+
"segment_count_current": 2127,
|
| 23 |
+
"segment_count_peak": 2127,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:35.910251Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75906373632,
|
| 28 |
+
"allocated_bytes_current_delta": 32348899840,
|
| 29 |
+
"allocated_bytes_peak": 82622775808,
|
| 30 |
+
"allocated_bytes_peak_delta": 39059010560,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43557473792,
|
| 33 |
+
"active_bytes_peak": 43563765248,
|
| 34 |
+
"allocated_bytes_current": 43557473792,
|
| 35 |
+
"allocated_bytes_peak": 43563765248,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1223012864,
|
| 38 |
+
"inactive_split_bytes_peak": 1612261376,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 44795166720,
|
| 46 |
+
"reserved_bytes_peak": 44795166720,
|
| 47 |
+
"segment_count_current": 887,
|
| 48 |
+
"segment_count_peak": 887,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:09.289271Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5924497408,
|
| 55 |
+
"inactive_split_bytes_current_delta": 4701484544,
|
| 56 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 12517172224,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 83022053376,
|
| 68 |
+
"reserved_bytes_current_delta": 38226886656,
|
| 69 |
+
"reserved_bytes_peak": 83022053376,
|
| 70 |
+
"reserved_bytes_peak_delta": 38226886656,
|
| 71 |
+
"segment_count_current": 2127,
|
| 72 |
+
"segment_count_current_delta": 1240,
|
| 73 |
+
"segment_count_peak": 2127,
|
| 74 |
+
"segment_count_peak_delta": 1240,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:36.045282Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5b3a54f0eb6432a48d3702bccd3e8e9f0d611e5cb2449b369de98b9d04d56876
|
| 3 |
+
size 7018235
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75906373632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113460826112,
|
| 5 |
+
"active_bytes_peak_delta": 30838050304,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75906373632,
|
| 8 |
+
"active_bytes_peak": 113460826112,
|
| 9 |
+
"allocated_bytes_current": 75906373632,
|
| 10 |
+
"allocated_bytes_peak": 113460826112,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 5926594560,
|
| 13 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 113862770688,
|
| 21 |
+
"reserved_bytes_peak": 113862770688,
|
| 22 |
+
"segment_count_current": 2841,
|
| 23 |
+
"segment_count_peak": 2841,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:37.348664Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75906373632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113460826112,
|
| 30 |
+
"allocated_bytes_peak_delta": 30838050304,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75906373632,
|
| 33 |
+
"active_bytes_peak": 82622775808,
|
| 34 |
+
"allocated_bytes_current": 75906373632,
|
| 35 |
+
"allocated_bytes_peak": 82622775808,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 5924497408,
|
| 38 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 83022053376,
|
| 46 |
+
"reserved_bytes_peak": 83022053376,
|
| 47 |
+
"segment_count_current": 2127,
|
| 48 |
+
"segment_count_peak": 2127,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:36.066185Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5926594560,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113862770688,
|
| 68 |
+
"reserved_bytes_current_delta": 30840717312,
|
| 69 |
+
"reserved_bytes_peak": 113862770688,
|
| 70 |
+
"reserved_bytes_peak_delta": 30840717312,
|
| 71 |
+
"segment_count_current": 2841,
|
| 72 |
+
"segment_count_current_delta": 714,
|
| 73 |
+
"segment_count_peak": 2841,
|
| 74 |
+
"segment_count_peak_delta": 714,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:37.477356Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b66639ce8538a0bb11755f6ccfcdb7faa6fabbb3ce191fd07e1245df4e09489c
|
| 3 |
+
size 6806262
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75906373632,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113474031616,
|
| 5 |
+
"active_bytes_peak_delta": 13205504,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75906373632,
|
| 8 |
+
"active_bytes_peak": 113474031616,
|
| 9 |
+
"allocated_bytes_current": 75906373632,
|
| 10 |
+
"allocated_bytes_peak": 113474031616,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 5924497408,
|
| 13 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 113862770688,
|
| 21 |
+
"reserved_bytes_peak": 113862770688,
|
| 22 |
+
"segment_count_current": 2841,
|
| 23 |
+
"segment_count_peak": 2841,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:38.517239Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75906373632,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113474031616,
|
| 30 |
+
"allocated_bytes_peak_delta": 13205504,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75906373632,
|
| 33 |
+
"active_bytes_peak": 113460826112,
|
| 34 |
+
"allocated_bytes_current": 75906373632,
|
| 35 |
+
"allocated_bytes_peak": 113460826112,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 5926594560,
|
| 38 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 113862770688,
|
| 46 |
+
"reserved_bytes_peak": 113862770688,
|
| 47 |
+
"segment_count_current": 2841,
|
| 48 |
+
"segment_count_peak": 2841,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:37.492920Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5924497408,
|
| 55 |
+
"inactive_split_bytes_current_delta": -2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113862770688,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113862770688,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2841,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2841,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:38.645475Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e8efaadcf1709cdc1b13914564edcdc3e5346f5010e53e50c204a57c22acdbd1
|
| 3 |
+
size 6771814
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75923417088,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113474031616,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75923417088,
|
| 8 |
+
"active_bytes_peak": 113474031616,
|
| 9 |
+
"allocated_bytes_current": 75923417088,
|
| 10 |
+
"allocated_bytes_peak": 113474031616,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 5961979904,
|
| 13 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 119059513344,
|
| 21 |
+
"reserved_bytes_peak": 119059513344,
|
| 22 |
+
"segment_count_current": 3136,
|
| 23 |
+
"segment_count_peak": 3136,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:41.821350Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75923417088,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113474031616,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75906373632,
|
| 33 |
+
"active_bytes_peak": 113474031616,
|
| 34 |
+
"allocated_bytes_current": 75906373632,
|
| 35 |
+
"allocated_bytes_peak": 113474031616,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 5924497408,
|
| 38 |
+
"inactive_split_bytes_peak": 14129433600,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 113862770688,
|
| 46 |
+
"reserved_bytes_peak": 113862770688,
|
| 47 |
+
"segment_count_current": 2841,
|
| 48 |
+
"segment_count_peak": 2841,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:38.660609Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5961979904,
|
| 55 |
+
"inactive_split_bytes_current_delta": 37482496,
|
| 56 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1349727744,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119059513344,
|
| 68 |
+
"reserved_bytes_current_delta": 5196742656,
|
| 69 |
+
"reserved_bytes_peak": 119059513344,
|
| 70 |
+
"reserved_bytes_peak_delta": 5196742656,
|
| 71 |
+
"segment_count_current": 3136,
|
| 72 |
+
"segment_count_current_delta": 295,
|
| 73 |
+
"segment_count_peak": 3136,
|
| 74 |
+
"segment_count_peak_delta": 295,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:41.963989Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cec3c71cc3c2b7963ee074aebfd39e9437869d8a12b22312c2c2b052a35902bc
|
| 3 |
+
size 7466230
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75923417088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113474031616,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75923417088,
|
| 8 |
+
"active_bytes_peak": 113474031616,
|
| 9 |
+
"allocated_bytes_current": 75923417088,
|
| 10 |
+
"allocated_bytes_peak": 113474031616,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 5961979904,
|
| 13 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 119059513344,
|
| 21 |
+
"reserved_bytes_peak": 119059513344,
|
| 22 |
+
"segment_count_current": 3136,
|
| 23 |
+
"segment_count_peak": 3136,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.351569Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75923417088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113474031616,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75923417088,
|
| 33 |
+
"active_bytes_peak": 113474031616,
|
| 34 |
+
"allocated_bytes_current": 75923417088,
|
| 35 |
+
"allocated_bytes_peak": 113474031616,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 5961979904,
|
| 38 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 119059513344,
|
| 46 |
+
"reserved_bytes_peak": 119059513344,
|
| 47 |
+
"segment_count_current": 3136,
|
| 48 |
+
"segment_count_peak": 3136,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:41.980683Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5961979904,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119059513344,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119059513344,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3136,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3136,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.372512Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a8c8275abfb2f2937daa8e5310c30653f639027b80d6cd792edc076f091751a4
|
| 3 |
+
size 901220
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75923417088,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113474031616,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75923417088,
|
| 8 |
+
"active_bytes_peak": 113474031616,
|
| 9 |
+
"allocated_bytes_current": 75923417088,
|
| 10 |
+
"allocated_bytes_peak": 113474031616,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 5961979904,
|
| 13 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 119059513344,
|
| 21 |
+
"reserved_bytes_peak": 119059513344,
|
| 22 |
+
"segment_count_current": 3136,
|
| 23 |
+
"segment_count_peak": 3136,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.732501Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75923417088,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113474031616,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75923417088,
|
| 33 |
+
"active_bytes_peak": 113474031616,
|
| 34 |
+
"allocated_bytes_current": 75923417088,
|
| 35 |
+
"allocated_bytes_peak": 113474031616,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 5961979904,
|
| 38 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 119059513344,
|
| 46 |
+
"reserved_bytes_peak": 119059513344,
|
| 47 |
+
"segment_count_current": 3136,
|
| 48 |
+
"segment_count_peak": 3136,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:42.374142Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5961979904,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15479161344,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119059513344,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119059513344,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3136,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3136,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.751336Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:70cf0db8ddacad01116ec4c48b801841bbb386f001903ce288f1f9829c621bff
|
| 3 |
+
size 901220
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75889472000,
|
| 3 |
+
"active_bytes_current_delta": 32338143232,
|
| 4 |
+
"active_bytes_peak": 82618485248,
|
| 5 |
+
"active_bytes_peak_delta": 39060865024,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75889472000,
|
| 8 |
+
"active_bytes_peak": 82618485248,
|
| 9 |
+
"allocated_bytes_current": 75889472000,
|
| 10 |
+
"allocated_bytes_peak": 82618485248,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 5903650304,
|
| 13 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 83001081856,
|
| 21 |
+
"reserved_bytes_peak": 83001081856,
|
| 22 |
+
"segment_count_current": 2127,
|
| 23 |
+
"segment_count_peak": 2127,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:35.910248Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75889472000,
|
| 28 |
+
"allocated_bytes_current_delta": 32338143232,
|
| 29 |
+
"allocated_bytes_peak": 82618485248,
|
| 30 |
+
"allocated_bytes_peak_delta": 39060865024,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43551328768,
|
| 33 |
+
"active_bytes_peak": 43557620224,
|
| 34 |
+
"allocated_bytes_current": 43551328768,
|
| 35 |
+
"allocated_bytes_peak": 43557620224,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1254323712,
|
| 38 |
+
"inactive_split_bytes_peak": 1612261376,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 44820332544,
|
| 46 |
+
"reserved_bytes_peak": 44820332544,
|
| 47 |
+
"segment_count_current": 889,
|
| 48 |
+
"segment_count_peak": 889,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:09.289064Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5903650304,
|
| 55 |
+
"inactive_split_bytes_current_delta": 4649326592,
|
| 56 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 12523649024,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 83001081856,
|
| 68 |
+
"reserved_bytes_current_delta": 38180749312,
|
| 69 |
+
"reserved_bytes_peak": 83001081856,
|
| 70 |
+
"reserved_bytes_peak_delta": 38180749312,
|
| 71 |
+
"segment_count_current": 2127,
|
| 72 |
+
"segment_count_current_delta": 1238,
|
| 73 |
+
"segment_count_peak": 2127,
|
| 74 |
+
"segment_count_peak_delta": 1238,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:36.047700Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:938c759b9247f4d8c27dc09ea87f60048674572c7c241818b3c846af4811b62d
|
| 3 |
+
size 7019264
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75889472000,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113449531904,
|
| 5 |
+
"active_bytes_peak_delta": 30831046656,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75889472000,
|
| 8 |
+
"active_bytes_peak": 113449531904,
|
| 9 |
+
"allocated_bytes_current": 75889472000,
|
| 10 |
+
"allocated_bytes_peak": 113449531904,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 5903650304,
|
| 13 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 113894227968,
|
| 21 |
+
"reserved_bytes_peak": 113894227968,
|
| 22 |
+
"segment_count_current": 2840,
|
| 23 |
+
"segment_count_peak": 2840,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:37.348755Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75889472000,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113449531904,
|
| 30 |
+
"allocated_bytes_peak_delta": 30831046656,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75889472000,
|
| 33 |
+
"active_bytes_peak": 82618485248,
|
| 34 |
+
"allocated_bytes_current": 75889472000,
|
| 35 |
+
"allocated_bytes_peak": 82618485248,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 5903650304,
|
| 38 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 83001081856,
|
| 46 |
+
"reserved_bytes_peak": 83001081856,
|
| 47 |
+
"segment_count_current": 2127,
|
| 48 |
+
"segment_count_peak": 2127,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:36.068725Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5903650304,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113894227968,
|
| 68 |
+
"reserved_bytes_current_delta": 30893146112,
|
| 69 |
+
"reserved_bytes_peak": 113894227968,
|
| 70 |
+
"reserved_bytes_peak_delta": 30893146112,
|
| 71 |
+
"segment_count_current": 2840,
|
| 72 |
+
"segment_count_current_delta": 713,
|
| 73 |
+
"segment_count_peak": 2840,
|
| 74 |
+
"segment_count_peak_delta": 713,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:37.478575Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:17a59db31fcb1178c6c17f0052dcce04b3b1b625978b1186d12783ae15ce5696
|
| 3 |
+
size 6887605
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75889472000,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113484609024,
|
| 5 |
+
"active_bytes_peak_delta": 35077120,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75889472000,
|
| 8 |
+
"active_bytes_peak": 113484609024,
|
| 9 |
+
"allocated_bytes_current": 75889472000,
|
| 10 |
+
"allocated_bytes_peak": 113484609024,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 5903650304,
|
| 13 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 113894227968,
|
| 21 |
+
"reserved_bytes_peak": 113894227968,
|
| 22 |
+
"segment_count_current": 2840,
|
| 23 |
+
"segment_count_peak": 2840,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:38.517318Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75889472000,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113484609024,
|
| 30 |
+
"allocated_bytes_peak_delta": 35077120,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75889472000,
|
| 33 |
+
"active_bytes_peak": 113449531904,
|
| 34 |
+
"allocated_bytes_current": 75889472000,
|
| 35 |
+
"allocated_bytes_peak": 113449531904,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 5903650304,
|
| 38 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 113894227968,
|
| 46 |
+
"reserved_bytes_peak": 113894227968,
|
| 47 |
+
"segment_count_current": 2840,
|
| 48 |
+
"segment_count_peak": 2840,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:37.494110Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5903650304,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113894227968,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 113894227968,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2840,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2840,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:38.650019Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b915e4bfb453cd576b0d714de462afc17aeec39ebeec84f050668f313c22dcc9
|
| 3 |
+
size 6853737
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75906515456,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 113484609024,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75906515456,
|
| 8 |
+
"active_bytes_peak": 113484609024,
|
| 9 |
+
"allocated_bytes_current": 75906515456,
|
| 10 |
+
"allocated_bytes_peak": 113484609024,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 5941132800,
|
| 13 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 119021764608,
|
| 21 |
+
"reserved_bytes_peak": 119021764608,
|
| 22 |
+
"segment_count_current": 3128,
|
| 23 |
+
"segment_count_peak": 3128,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:41.821352Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75906515456,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 113484609024,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75889472000,
|
| 33 |
+
"active_bytes_peak": 113484609024,
|
| 34 |
+
"allocated_bytes_current": 75889472000,
|
| 35 |
+
"allocated_bytes_peak": 113484609024,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 5903650304,
|
| 38 |
+
"inactive_split_bytes_peak": 14135910400,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 113894227968,
|
| 46 |
+
"reserved_bytes_peak": 113894227968,
|
| 47 |
+
"segment_count_current": 2840,
|
| 48 |
+
"segment_count_peak": 2840,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:38.664950Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5941132800,
|
| 55 |
+
"inactive_split_bytes_current_delta": 37482496,
|
| 56 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1322403840,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119021764608,
|
| 68 |
+
"reserved_bytes_current_delta": 5127536640,
|
| 69 |
+
"reserved_bytes_peak": 119021764608,
|
| 70 |
+
"reserved_bytes_peak_delta": 5127536640,
|
| 71 |
+
"segment_count_current": 3128,
|
| 72 |
+
"segment_count_current_delta": 288,
|
| 73 |
+
"segment_count_peak": 3128,
|
| 74 |
+
"segment_count_peak_delta": 288,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:41.966789Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2dad172e0d2c6dc1c5c2a863553b608c87e1edb0f663f8f11233e102566d2f0f
|
| 3 |
+
size 7509462
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75906515456,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113484609024,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75906515456,
|
| 8 |
+
"active_bytes_peak": 113484609024,
|
| 9 |
+
"allocated_bytes_current": 75906515456,
|
| 10 |
+
"allocated_bytes_peak": 113484609024,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 5941132800,
|
| 13 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 119021764608,
|
| 21 |
+
"reserved_bytes_peak": 119021764608,
|
| 22 |
+
"segment_count_current": 3128,
|
| 23 |
+
"segment_count_peak": 3128,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.351542Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75906515456,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113484609024,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75906515456,
|
| 33 |
+
"active_bytes_peak": 113484609024,
|
| 34 |
+
"allocated_bytes_current": 75906515456,
|
| 35 |
+
"allocated_bytes_peak": 113484609024,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 5941132800,
|
| 38 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 119021764608,
|
| 46 |
+
"reserved_bytes_peak": 119021764608,
|
| 47 |
+
"segment_count_current": 3128,
|
| 48 |
+
"segment_count_peak": 3128,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:41.987871Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5941132800,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119021764608,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119021764608,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3128,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3128,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.372555Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bfc88224475f580c8895dcd920a7e741377540aa089ea53514dc70213fe34d75
|
| 3 |
+
size 903196
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75906515456,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113484609024,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75906515456,
|
| 8 |
+
"active_bytes_peak": 113484609024,
|
| 9 |
+
"allocated_bytes_current": 75906515456,
|
| 10 |
+
"allocated_bytes_peak": 113484609024,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 5941132800,
|
| 13 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 119021764608,
|
| 21 |
+
"reserved_bytes_peak": 119021764608,
|
| 22 |
+
"segment_count_current": 3128,
|
| 23 |
+
"segment_count_peak": 3128,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:42.732549Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75906515456,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113484609024,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75906515456,
|
| 33 |
+
"active_bytes_peak": 113484609024,
|
| 34 |
+
"allocated_bytes_current": 75906515456,
|
| 35 |
+
"allocated_bytes_peak": 113484609024,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 5941132800,
|
| 38 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 119021764608,
|
| 46 |
+
"reserved_bytes_peak": 119021764608,
|
| 47 |
+
"segment_count_current": 3128,
|
| 48 |
+
"segment_count_peak": 3128,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:42.374192Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5941132800,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 15458314240,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119021764608,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119021764608,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3128,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3128,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:42.751576Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:860eb9dcd108badc7a929a9055d4932e004bca7ae75c76ec1aaf6dc2dd7e394f
|
| 3 |
+
size 903196
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75883125760,
|
| 3 |
+
"active_bytes_current_delta": 32328931328,
|
| 4 |
+
"active_bytes_peak": 82624266240,
|
| 5 |
+
"active_bytes_peak_delta": 39063780352,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75883125760,
|
| 8 |
+
"active_bytes_peak": 82624266240,
|
| 9 |
+
"allocated_bytes_current": 75883125760,
|
| 10 |
+
"allocated_bytes_peak": 82624266240,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 5901607936,
|
| 13 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 82996887552,
|
| 21 |
+
"reserved_bytes_peak": 82996887552,
|
| 22 |
+
"segment_count_current": 2112,
|
| 23 |
+
"segment_count_peak": 2112,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:35.910208Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75883125760,
|
| 28 |
+
"allocated_bytes_current_delta": 32328931328,
|
| 29 |
+
"allocated_bytes_peak": 82624266240,
|
| 30 |
+
"allocated_bytes_peak_delta": 39063780352,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43554194432,
|
| 33 |
+
"active_bytes_peak": 43560485888,
|
| 34 |
+
"allocated_bytes_current": 43554194432,
|
| 35 |
+
"allocated_bytes_peak": 43560485888,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1255652352,
|
| 38 |
+
"inactive_split_bytes_peak": 1612261376,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 44835012608,
|
| 46 |
+
"reserved_bytes_peak": 44835012608,
|
| 47 |
+
"segment_count_current": 883,
|
| 48 |
+
"segment_count_peak": 883,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:09.289231Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5901607936,
|
| 55 |
+
"inactive_split_bytes_current_delta": 4645955584,
|
| 56 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 12537032192,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 82996887552,
|
| 68 |
+
"reserved_bytes_current_delta": 38161874944,
|
| 69 |
+
"reserved_bytes_peak": 82996887552,
|
| 70 |
+
"reserved_bytes_peak_delta": 38161874944,
|
| 71 |
+
"segment_count_current": 2112,
|
| 72 |
+
"segment_count_current_delta": 1229,
|
| 73 |
+
"segment_count_peak": 2112,
|
| 74 |
+
"segment_count_peak_delta": 1229,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:36.046760Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1873ea447f9791d697e95e1b4efbbaf57a09fbc11a968d7d1450883233f61756
|
| 3 |
+
size 7001386
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 75883125760,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 113455617536,
|
| 5 |
+
"active_bytes_peak_delta": 30831351296,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 75883125760,
|
| 8 |
+
"active_bytes_peak": 113455617536,
|
| 9 |
+
"allocated_bytes_current": 75883125760,
|
| 10 |
+
"allocated_bytes_peak": 113455617536,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 5901607936,
|
| 13 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 113841799168,
|
| 21 |
+
"reserved_bytes_peak": 113841799168,
|
| 22 |
+
"segment_count_current": 2812,
|
| 23 |
+
"segment_count_peak": 2812,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:17:37.348746Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 75883125760,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 113455617536,
|
| 30 |
+
"allocated_bytes_peak_delta": 30831351296,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 75883125760,
|
| 33 |
+
"active_bytes_peak": 82624266240,
|
| 34 |
+
"allocated_bytes_current": 75883125760,
|
| 35 |
+
"allocated_bytes_peak": 82624266240,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 5901607936,
|
| 38 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 82996887552,
|
| 46 |
+
"reserved_bytes_peak": 82996887552,
|
| 47 |
+
"segment_count_current": 2112,
|
| 48 |
+
"segment_count_peak": 2112,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:17:36.066321Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 5901607936,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14149293568,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 113841799168,
|
| 68 |
+
"reserved_bytes_current_delta": 30844911616,
|
| 69 |
+
"reserved_bytes_peak": 113841799168,
|
| 70 |
+
"reserved_bytes_peak_delta": 30844911616,
|
| 71 |
+
"segment_count_current": 2812,
|
| 72 |
+
"segment_count_current_delta": 700,
|
| 73 |
+
"segment_count_peak": 2812,
|
| 74 |
+
"segment_count_peak_delta": 700,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:17:37.477811Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6bd4983fbce8115d5dfac2488b6a014fa7ba53a90610645caf927c4081abcbe3
|
| 3 |
+
size 6492481
|