granularity results ultraep/tierbmem @ 2026-08-26T19:31:45Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env +61 -0
- ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log +0 -0
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38993883648,
|
| 3 |
+
"active_bytes_current_delta": 15945322496,
|
| 4 |
+
"active_bytes_peak": 109588680192,
|
| 5 |
+
"active_bytes_peak_delta": 86527536128,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38993883648,
|
| 8 |
+
"active_bytes_peak": 109588680192,
|
| 9 |
+
"allocated_bytes_current": 38993883648,
|
| 10 |
+
"allocated_bytes_peak": 109588680192,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 14059867648,
|
| 13 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 112956801024,
|
| 21 |
+
"reserved_bytes_peak": 112956801024,
|
| 22 |
+
"segment_count_current": 847,
|
| 23 |
+
"segment_count_peak": 847,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.258808Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38993883648,
|
| 28 |
+
"allocated_bytes_current_delta": 15945322496,
|
| 29 |
+
"allocated_bytes_peak": 109588680192,
|
| 30 |
+
"allocated_bytes_peak_delta": 86527536128,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23048561152,
|
| 33 |
+
"active_bytes_peak": 23061144064,
|
| 34 |
+
"allocated_bytes_current": 23048561152,
|
| 35 |
+
"allocated_bytes_peak": 23061144064,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 389209600,
|
| 38 |
+
"inactive_split_bytes_peak": 883057664,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 25623003136,
|
| 46 |
+
"reserved_bytes_peak": 25623003136,
|
| 47 |
+
"segment_count_current": 551,
|
| 48 |
+
"segment_count_peak": 551,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:17.758985Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 14059867648,
|
| 55 |
+
"inactive_split_bytes_current_delta": 13670658048,
|
| 56 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 26963017216,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 112956801024,
|
| 68 |
+
"reserved_bytes_current_delta": 87333797888,
|
| 69 |
+
"reserved_bytes_peak": 112956801024,
|
| 70 |
+
"reserved_bytes_peak_delta": 87333797888,
|
| 71 |
+
"segment_count_current": 847,
|
| 72 |
+
"segment_count_current_delta": 296,
|
| 73 |
+
"segment_count_peak": 847,
|
| 74 |
+
"segment_count_peak_delta": 296,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.320281Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:83a7120506693b5adcf9d91a1bca38a7b521d79c9f2046a36149f5c375e2c804
|
| 3 |
+
size 3054015
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38993883648,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123802562048,
|
| 5 |
+
"active_bytes_peak_delta": 14213881856,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38993883648,
|
| 8 |
+
"active_bytes_peak": 123802562048,
|
| 9 |
+
"allocated_bytes_current": 38993883648,
|
| 10 |
+
"allocated_bytes_peak": 123802562048,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 14059867648,
|
| 13 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 127500550144,
|
| 21 |
+
"reserved_bytes_peak": 127500550144,
|
| 22 |
+
"segment_count_current": 859,
|
| 23 |
+
"segment_count_peak": 859,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.929126Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38993883648,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123802562048,
|
| 30 |
+
"allocated_bytes_peak_delta": 14213881856,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38993883648,
|
| 33 |
+
"active_bytes_peak": 109588680192,
|
| 34 |
+
"allocated_bytes_current": 38993883648,
|
| 35 |
+
"allocated_bytes_peak": 109588680192,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 14059867648,
|
| 38 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 112956801024,
|
| 46 |
+
"reserved_bytes_peak": 112956801024,
|
| 47 |
+
"segment_count_current": 847,
|
| 48 |
+
"segment_count_peak": 847,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.329151Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 14059867648,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127500550144,
|
| 68 |
+
"reserved_bytes_current_delta": 14543749120,
|
| 69 |
+
"reserved_bytes_peak": 127500550144,
|
| 70 |
+
"reserved_bytes_peak_delta": 14543749120,
|
| 71 |
+
"segment_count_current": 859,
|
| 72 |
+
"segment_count_current_delta": 12,
|
| 73 |
+
"segment_count_peak": 859,
|
| 74 |
+
"segment_count_peak_delta": 12,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.980787Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cc4f989a013e267c2cfaf9ce568bff5374f09480dedd764567cb3e2664e09bdb
|
| 3 |
+
size 3049002
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38993883648,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123803930112,
|
| 5 |
+
"active_bytes_peak_delta": 1368064,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38993883648,
|
| 8 |
+
"active_bytes_peak": 123803930112,
|
| 9 |
+
"allocated_bytes_current": 38993883648,
|
| 10 |
+
"allocated_bytes_peak": 123803930112,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 14059867648,
|
| 13 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 127500550144,
|
| 21 |
+
"reserved_bytes_peak": 127500550144,
|
| 22 |
+
"segment_count_current": 859,
|
| 23 |
+
"segment_count_peak": 859,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:46.587185Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38993883648,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123803930112,
|
| 30 |
+
"allocated_bytes_peak_delta": 1368064,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38993883648,
|
| 33 |
+
"active_bytes_peak": 123802562048,
|
| 34 |
+
"allocated_bytes_current": 38993883648,
|
| 35 |
+
"allocated_bytes_peak": 123802562048,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 14059867648,
|
| 38 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 127500550144,
|
| 46 |
+
"reserved_bytes_peak": 127500550144,
|
| 47 |
+
"segment_count_current": 859,
|
| 48 |
+
"segment_count_peak": 859,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.986674Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 14059867648,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127500550144,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 127500550144,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 859,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 859,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:46.638228Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a31cb9f06e39cd018076e2932e645017551c8ecdf23505ffa7b8d84aae3810c2
|
| 3 |
+
size 3065016
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 39011844608,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 123803930112,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 39011844608,
|
| 8 |
+
"active_bytes_peak": 123803930112,
|
| 9 |
+
"allocated_bytes_current": 39011844608,
|
| 10 |
+
"allocated_bytes_peak": 123803930112,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 16214556160,
|
| 13 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 140314148864,
|
| 21 |
+
"reserved_bytes_peak": 140314148864,
|
| 22 |
+
"segment_count_current": 970,
|
| 23 |
+
"segment_count_peak": 970,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:48.442889Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 39011844608,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 123803930112,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38993883648,
|
| 33 |
+
"active_bytes_peak": 123803930112,
|
| 34 |
+
"allocated_bytes_current": 38993883648,
|
| 35 |
+
"allocated_bytes_peak": 123803930112,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 14059867648,
|
| 38 |
+
"inactive_split_bytes_peak": 27846074880,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 127500550144,
|
| 46 |
+
"reserved_bytes_peak": 127500550144,
|
| 47 |
+
"segment_count_current": 859,
|
| 48 |
+
"segment_count_peak": 859,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:46.644129Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16214556160,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2154688512,
|
| 56 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 11306130944,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140314148864,
|
| 68 |
+
"reserved_bytes_current_delta": 12813598720,
|
| 69 |
+
"reserved_bytes_peak": 140314148864,
|
| 70 |
+
"reserved_bytes_peak_delta": 12813598720,
|
| 71 |
+
"segment_count_current": 970,
|
| 72 |
+
"segment_count_current_delta": 111,
|
| 73 |
+
"segment_count_peak": 970,
|
| 74 |
+
"segment_count_peak_delta": 111,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:48.501132Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:17efcabdc174de09d2a136bf51ce42521716d10ed82c4020b784e72e341b1763
|
| 3 |
+
size 3226224
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 39011844608,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123803930112,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 39011844608,
|
| 8 |
+
"active_bytes_peak": 123803930112,
|
| 9 |
+
"allocated_bytes_current": 39011844608,
|
| 10 |
+
"allocated_bytes_peak": 123803930112,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 16214556160,
|
| 13 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 140314148864,
|
| 21 |
+
"reserved_bytes_peak": 140314148864,
|
| 22 |
+
"segment_count_current": 970,
|
| 23 |
+
"segment_count_peak": 970,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.089210Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 39011844608,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123803930112,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 39011844608,
|
| 33 |
+
"active_bytes_peak": 123803930112,
|
| 34 |
+
"allocated_bytes_current": 39011844608,
|
| 35 |
+
"allocated_bytes_peak": 123803930112,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 16214556160,
|
| 38 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 140314148864,
|
| 46 |
+
"reserved_bytes_peak": 140314148864,
|
| 47 |
+
"segment_count_current": 970,
|
| 48 |
+
"segment_count_peak": 970,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:48.508069Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16214556160,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140314148864,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 140314148864,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 970,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 970,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.097417Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cec6e5732ef24b557f71d1208d1d83a838a2c444e47941d558eb4b8c6d85b91a
|
| 3 |
+
size 370864
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 39011844608,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123803930112,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 39011844608,
|
| 8 |
+
"active_bytes_peak": 123803930112,
|
| 9 |
+
"allocated_bytes_current": 39011844608,
|
| 10 |
+
"allocated_bytes_peak": 123803930112,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 16214556160,
|
| 13 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 140314148864,
|
| 21 |
+
"reserved_bytes_peak": 140314148864,
|
| 22 |
+
"segment_count_current": 970,
|
| 23 |
+
"segment_count_peak": 970,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.684045Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 39011844608,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123803930112,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 39011844608,
|
| 33 |
+
"active_bytes_peak": 123803930112,
|
| 34 |
+
"allocated_bytes_current": 39011844608,
|
| 35 |
+
"allocated_bytes_peak": 123803930112,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 16214556160,
|
| 38 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 140314148864,
|
| 46 |
+
"reserved_bytes_peak": 140314148864,
|
| 47 |
+
"segment_count_current": 970,
|
| 48 |
+
"segment_count_peak": 970,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:49.098450Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16214556160,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 39152205824,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140314148864,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 140314148864,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 970,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 970,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.692107Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:50d5a1c42eda960f09f3c2d51a8efd01aafd7a1c7a2f33260dd3fb2f9ea3d71a
|
| 3 |
+
size 370864
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38984414720,
|
| 3 |
+
"active_bytes_current_delta": 15931054080,
|
| 4 |
+
"active_bytes_peak": 109604034048,
|
| 5 |
+
"active_bytes_peak_delta": 86538090496,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38984414720,
|
| 8 |
+
"active_bytes_peak": 109604034048,
|
| 9 |
+
"allocated_bytes_current": 38984414720,
|
| 10 |
+
"allocated_bytes_peak": 109604034048,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 13970770432,
|
| 13 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 112885497856,
|
| 21 |
+
"reserved_bytes_peak": 112885497856,
|
| 22 |
+
"segment_count_current": 844,
|
| 23 |
+
"segment_count_peak": 844,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.258776Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38984414720,
|
| 28 |
+
"allocated_bytes_current_delta": 15931054080,
|
| 29 |
+
"allocated_bytes_peak": 109604034048,
|
| 30 |
+
"allocated_bytes_peak_delta": 86538090496,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23053360640,
|
| 33 |
+
"active_bytes_peak": 23065943552,
|
| 34 |
+
"allocated_bytes_current": 23053360640,
|
| 35 |
+
"allocated_bytes_peak": 23065943552,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 474587648,
|
| 38 |
+
"inactive_split_bytes_peak": 883057664,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 25623003136,
|
| 46 |
+
"reserved_bytes_peak": 25623003136,
|
| 47 |
+
"segment_count_current": 551,
|
| 48 |
+
"segment_count_peak": 551,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:17.759118Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 13970770432,
|
| 55 |
+
"inactive_split_bytes_current_delta": 13496182784,
|
| 56 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 26933526016,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 112885497856,
|
| 68 |
+
"reserved_bytes_current_delta": 87262494720,
|
| 69 |
+
"reserved_bytes_peak": 112885497856,
|
| 70 |
+
"reserved_bytes_peak_delta": 87262494720,
|
| 71 |
+
"segment_count_current": 844,
|
| 72 |
+
"segment_count_current_delta": 293,
|
| 73 |
+
"segment_count_peak": 844,
|
| 74 |
+
"segment_count_peak_delta": 293,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.319604Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9341d9d5fa4db59064b5f59b8a55b4c55fde01c00f22771fd9bf802548f669d5
|
| 3 |
+
size 3011418
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38984414720,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123775942144,
|
| 5 |
+
"active_bytes_peak_delta": 14171908096,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38984414720,
|
| 8 |
+
"active_bytes_peak": 123775942144,
|
| 9 |
+
"allocated_bytes_current": 38984414720,
|
| 10 |
+
"allocated_bytes_peak": 123775942144,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 13970770432,
|
| 13 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 127425052672,
|
| 21 |
+
"reserved_bytes_peak": 127425052672,
|
| 22 |
+
"segment_count_current": 854,
|
| 23 |
+
"segment_count_peak": 854,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.929130Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38984414720,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123775942144,
|
| 30 |
+
"allocated_bytes_peak_delta": 14171908096,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38984414720,
|
| 33 |
+
"active_bytes_peak": 109604034048,
|
| 34 |
+
"allocated_bytes_current": 38984414720,
|
| 35 |
+
"allocated_bytes_peak": 109604034048,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 13970770432,
|
| 38 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 112885497856,
|
| 46 |
+
"reserved_bytes_peak": 112885497856,
|
| 47 |
+
"segment_count_current": 844,
|
| 48 |
+
"segment_count_peak": 844,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.328437Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 13970770432,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127425052672,
|
| 68 |
+
"reserved_bytes_current_delta": 14539554816,
|
| 69 |
+
"reserved_bytes_peak": 127425052672,
|
| 70 |
+
"reserved_bytes_peak_delta": 14539554816,
|
| 71 |
+
"segment_count_current": 854,
|
| 72 |
+
"segment_count_current_delta": 10,
|
| 73 |
+
"segment_count_peak": 854,
|
| 74 |
+
"segment_count_peak_delta": 10,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.980410Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6ed801a69e72e5224d35aa1d873046563c8e396d2748ad13c4ea89e09b92a6a6
|
| 3 |
+
size 2977330
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38984414720,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123783904768,
|
| 5 |
+
"active_bytes_peak_delta": 7962624,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38984414720,
|
| 8 |
+
"active_bytes_peak": 123783904768,
|
| 9 |
+
"allocated_bytes_current": 38984414720,
|
| 10 |
+
"allocated_bytes_peak": 123783904768,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 13970770432,
|
| 13 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 127425052672,
|
| 21 |
+
"reserved_bytes_peak": 127425052672,
|
| 22 |
+
"segment_count_current": 854,
|
| 23 |
+
"segment_count_peak": 854,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:46.587182Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38984414720,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123783904768,
|
| 30 |
+
"allocated_bytes_peak_delta": 7962624,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38984414720,
|
| 33 |
+
"active_bytes_peak": 123775942144,
|
| 34 |
+
"allocated_bytes_current": 38984414720,
|
| 35 |
+
"allocated_bytes_peak": 123775942144,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 13970770432,
|
| 38 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 127425052672,
|
| 46 |
+
"reserved_bytes_peak": 127425052672,
|
| 47 |
+
"segment_count_current": 854,
|
| 48 |
+
"segment_count_peak": 854,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.986292Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 13970770432,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127425052672,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 127425052672,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 854,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 854,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:46.638582Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ddcccc20560180b7aa3d3f3772e82fdc11a13e38838401f8f6f5aec5b52d1e2f
|
| 3 |
+
size 2954732
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 39002375680,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 123783904768,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 39002375680,
|
| 8 |
+
"active_bytes_peak": 123783904768,
|
| 9 |
+
"allocated_bytes_current": 39002375680,
|
| 10 |
+
"allocated_bytes_peak": 123783904768,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 16125458944,
|
| 13 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 140215582720,
|
| 21 |
+
"reserved_bytes_peak": 140215582720,
|
| 22 |
+
"segment_count_current": 954,
|
| 23 |
+
"segment_count_peak": 954,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:48.442887Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 39002375680,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 123783904768,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38984414720,
|
| 33 |
+
"active_bytes_peak": 123783904768,
|
| 34 |
+
"allocated_bytes_current": 38984414720,
|
| 35 |
+
"allocated_bytes_peak": 123783904768,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 13970770432,
|
| 38 |
+
"inactive_split_bytes_peak": 27816583680,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 127425052672,
|
| 46 |
+
"reserved_bytes_peak": 127425052672,
|
| 47 |
+
"segment_count_current": 854,
|
| 48 |
+
"segment_count_peak": 854,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:46.644412Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16125458944,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2154688512,
|
| 56 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 11246524928,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140215582720,
|
| 68 |
+
"reserved_bytes_current_delta": 12790530048,
|
| 69 |
+
"reserved_bytes_peak": 140215582720,
|
| 70 |
+
"reserved_bytes_peak_delta": 12790530048,
|
| 71 |
+
"segment_count_current": 954,
|
| 72 |
+
"segment_count_current_delta": 100,
|
| 73 |
+
"segment_count_peak": 954,
|
| 74 |
+
"segment_count_peak_delta": 100,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:48.500293Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c4fb213e3645e5468f55929eff49a7c322a3e786f17fee99dfde3863d7ee5289
|
| 3 |
+
size 3107068
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 39002375680,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123783904768,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 39002375680,
|
| 8 |
+
"active_bytes_peak": 123783904768,
|
| 9 |
+
"allocated_bytes_current": 39002375680,
|
| 10 |
+
"allocated_bytes_peak": 123783904768,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 16125458944,
|
| 13 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 140215582720,
|
| 21 |
+
"reserved_bytes_peak": 140215582720,
|
| 22 |
+
"segment_count_current": 954,
|
| 23 |
+
"segment_count_peak": 954,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.089156Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 39002375680,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123783904768,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 39002375680,
|
| 33 |
+
"active_bytes_peak": 123783904768,
|
| 34 |
+
"allocated_bytes_current": 39002375680,
|
| 35 |
+
"allocated_bytes_peak": 123783904768,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 16125458944,
|
| 38 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 140215582720,
|
| 46 |
+
"reserved_bytes_peak": 140215582720,
|
| 47 |
+
"segment_count_current": 954,
|
| 48 |
+
"segment_count_peak": 954,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:48.507139Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16125458944,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140215582720,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 140215582720,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 954,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 954,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.097290Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:86ee3b1e2b4ba11de5e21e6070dd5cc3455d6c71933b42b8b11cb08a5089ce7a
|
| 3 |
+
size 342390
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 39002375680,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123783904768,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 39002375680,
|
| 8 |
+
"active_bytes_peak": 123783904768,
|
| 9 |
+
"allocated_bytes_current": 39002375680,
|
| 10 |
+
"allocated_bytes_peak": 123783904768,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 16125458944,
|
| 13 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 140215582720,
|
| 21 |
+
"reserved_bytes_peak": 140215582720,
|
| 22 |
+
"segment_count_current": 954,
|
| 23 |
+
"segment_count_peak": 954,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.684055Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 39002375680,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123783904768,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 39002375680,
|
| 33 |
+
"active_bytes_peak": 123783904768,
|
| 34 |
+
"allocated_bytes_current": 39002375680,
|
| 35 |
+
"allocated_bytes_peak": 123783904768,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 16125458944,
|
| 38 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 140215582720,
|
| 46 |
+
"reserved_bytes_peak": 140215582720,
|
| 47 |
+
"segment_count_current": 954,
|
| 48 |
+
"segment_count_peak": 954,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:49.098305Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16125458944,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 39063108608,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140215582720,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 140215582720,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 954,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 954,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.691896Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4e269c98f92de8a97b417be2464a4e702baebd152c70326d4eee25ebc3edef8f
|
| 3 |
+
size 342390
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38969498112,
|
| 3 |
+
"active_bytes_current_delta": 15923573248,
|
| 4 |
+
"active_bytes_peak": 109581087232,
|
| 5 |
+
"active_bytes_peak_delta": 86522579456,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38969498112,
|
| 8 |
+
"active_bytes_peak": 109581087232,
|
| 9 |
+
"allocated_bytes_current": 38969498112,
|
| 10 |
+
"allocated_bytes_peak": 109581087232,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 14163944960,
|
| 13 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 110880620544,
|
| 21 |
+
"reserved_bytes_peak": 110880620544,
|
| 22 |
+
"segment_count_current": 851,
|
| 23 |
+
"segment_count_peak": 851,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.258693Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38969498112,
|
| 28 |
+
"allocated_bytes_current_delta": 15923573248,
|
| 29 |
+
"allocated_bytes_peak": 109581087232,
|
| 30 |
+
"allocated_bytes_peak_delta": 86522579456,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23045924864,
|
| 33 |
+
"active_bytes_peak": 23058507776,
|
| 34 |
+
"allocated_bytes_current": 23045924864,
|
| 35 |
+
"allocated_bytes_peak": 23058507776,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 370874368,
|
| 38 |
+
"inactive_split_bytes_peak": 883057664,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 25761415168,
|
| 46 |
+
"reserved_bytes_peak": 25761415168,
|
| 47 |
+
"segment_count_current": 557,
|
| 48 |
+
"segment_count_peak": 557,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:17.759100Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 14163944960,
|
| 55 |
+
"inactive_split_bytes_current_delta": 13793070592,
|
| 56 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 26972653568,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 110880620544,
|
| 68 |
+
"reserved_bytes_current_delta": 85119205376,
|
| 69 |
+
"reserved_bytes_peak": 110880620544,
|
| 70 |
+
"reserved_bytes_peak_delta": 85119205376,
|
| 71 |
+
"segment_count_current": 851,
|
| 72 |
+
"segment_count_current_delta": 294,
|
| 73 |
+
"segment_count_peak": 851,
|
| 74 |
+
"segment_count_peak_delta": 294,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.318867Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ab9d452f6c8ec9d84dfc01c9f6afa1d0424918b0ff568ccb7d0cf88afb520f03
|
| 3 |
+
size 3059007
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38969498112,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123804231168,
|
| 5 |
+
"active_bytes_peak_delta": 14223143936,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38969498112,
|
| 8 |
+
"active_bytes_peak": 123804231168,
|
| 9 |
+
"allocated_bytes_current": 38969498112,
|
| 10 |
+
"allocated_bytes_peak": 123804231168,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 14163944960,
|
| 13 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 127641059328,
|
| 21 |
+
"reserved_bytes_peak": 127641059328,
|
| 22 |
+
"segment_count_current": 866,
|
| 23 |
+
"segment_count_peak": 866,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.929006Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38969498112,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123804231168,
|
| 30 |
+
"allocated_bytes_peak_delta": 14223143936,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38969498112,
|
| 33 |
+
"active_bytes_peak": 109581087232,
|
| 34 |
+
"allocated_bytes_current": 38969498112,
|
| 35 |
+
"allocated_bytes_peak": 109581087232,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 14163944960,
|
| 38 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 110880620544,
|
| 46 |
+
"reserved_bytes_peak": 110880620544,
|
| 47 |
+
"segment_count_current": 851,
|
| 48 |
+
"segment_count_peak": 851,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.327625Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 14163944960,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127641059328,
|
| 68 |
+
"reserved_bytes_current_delta": 16760438784,
|
| 69 |
+
"reserved_bytes_peak": 127641059328,
|
| 70 |
+
"reserved_bytes_peak_delta": 16760438784,
|
| 71 |
+
"segment_count_current": 866,
|
| 72 |
+
"segment_count_current_delta": 15,
|
| 73 |
+
"segment_count_peak": 866,
|
| 74 |
+
"segment_count_peak_delta": 15,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.980292Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e1e36a3bd906118e6570f5b6d01261ce332954e8ca22ee517fec097c83996e02
|
| 3 |
+
size 2973374
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38969498112,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123805369856,
|
| 5 |
+
"active_bytes_peak_delta": 1138688,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38969498112,
|
| 8 |
+
"active_bytes_peak": 123805369856,
|
| 9 |
+
"allocated_bytes_current": 38969498112,
|
| 10 |
+
"allocated_bytes_peak": 123805369856,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 14163944960,
|
| 13 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 127641059328,
|
| 21 |
+
"reserved_bytes_peak": 127641059328,
|
| 22 |
+
"segment_count_current": 866,
|
| 23 |
+
"segment_count_peak": 866,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:46.587092Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38969498112,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123805369856,
|
| 30 |
+
"allocated_bytes_peak_delta": 1138688,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38969498112,
|
| 33 |
+
"active_bytes_peak": 123804231168,
|
| 34 |
+
"allocated_bytes_current": 38969498112,
|
| 35 |
+
"allocated_bytes_peak": 123804231168,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 14163944960,
|
| 38 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 127641059328,
|
| 46 |
+
"reserved_bytes_peak": 127641059328,
|
| 47 |
+
"segment_count_current": 866,
|
| 48 |
+
"segment_count_peak": 866,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.986039Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 14163944960,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127641059328,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 127641059328,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 866,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 866,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:46.637391Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:842e23b86e4332c4ad01fd6a72563c2fb7ea3e22c09e87c938975ad7e18151a6
|
| 3 |
+
size 2972521
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38987459072,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 123805369856,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38987459072,
|
| 8 |
+
"active_bytes_peak": 123805369856,
|
| 9 |
+
"allocated_bytes_current": 38987459072,
|
| 10 |
+
"allocated_bytes_peak": 123805369856,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 16318633472,
|
| 13 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 140393840640,
|
| 21 |
+
"reserved_bytes_peak": 140393840640,
|
| 22 |
+
"segment_count_current": 979,
|
| 23 |
+
"segment_count_peak": 979,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:48.442740Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38987459072,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 123805369856,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38969498112,
|
| 33 |
+
"active_bytes_peak": 123805369856,
|
| 34 |
+
"allocated_bytes_current": 38969498112,
|
| 35 |
+
"allocated_bytes_peak": 123805369856,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 14163944960,
|
| 38 |
+
"inactive_split_bytes_peak": 27855711232,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 127641059328,
|
| 46 |
+
"reserved_bytes_peak": 127641059328,
|
| 47 |
+
"segment_count_current": 866,
|
| 48 |
+
"segment_count_peak": 866,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:46.643091Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16318633472,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2154688512,
|
| 56 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 11402669056,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140393840640,
|
| 68 |
+
"reserved_bytes_current_delta": 12752781312,
|
| 69 |
+
"reserved_bytes_peak": 140393840640,
|
| 70 |
+
"reserved_bytes_peak_delta": 12752781312,
|
| 71 |
+
"segment_count_current": 979,
|
| 72 |
+
"segment_count_current_delta": 113,
|
| 73 |
+
"segment_count_peak": 979,
|
| 74 |
+
"segment_count_peak_delta": 113,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:48.500266Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:88135e920b0d584c0c8bd4e63216f4476de7dd75c98632e43ac7c3e4dc526a16
|
| 3 |
+
size 3125976
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38987459072,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123805369856,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38987459072,
|
| 8 |
+
"active_bytes_peak": 123805369856,
|
| 9 |
+
"allocated_bytes_current": 38987459072,
|
| 10 |
+
"allocated_bytes_peak": 123805369856,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 16318633472,
|
| 13 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 140393840640,
|
| 21 |
+
"reserved_bytes_peak": 140393840640,
|
| 22 |
+
"segment_count_current": 979,
|
| 23 |
+
"segment_count_peak": 979,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.089066Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38987459072,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123805369856,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38987459072,
|
| 33 |
+
"active_bytes_peak": 123805369856,
|
| 34 |
+
"allocated_bytes_current": 38987459072,
|
| 35 |
+
"allocated_bytes_peak": 123805369856,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 16318633472,
|
| 38 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 140393840640,
|
| 46 |
+
"reserved_bytes_peak": 140393840640,
|
| 47 |
+
"segment_count_current": 979,
|
| 48 |
+
"segment_count_peak": 979,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:48.507012Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16318633472,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140393840640,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 140393840640,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 979,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 979,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.097344Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6335d55966a4a3dc0c04469505d09d68dcc1b64c470488b99ed688dbf506b084
|
| 3 |
+
size 356377
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38987459072,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123805369856,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38987459072,
|
| 8 |
+
"active_bytes_peak": 123805369856,
|
| 9 |
+
"allocated_bytes_current": 38987459072,
|
| 10 |
+
"allocated_bytes_peak": 123805369856,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 16318633472,
|
| 13 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 140393840640,
|
| 21 |
+
"reserved_bytes_peak": 140393840640,
|
| 22 |
+
"segment_count_current": 979,
|
| 23 |
+
"segment_count_peak": 979,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.683944Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38987459072,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123805369856,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38987459072,
|
| 33 |
+
"active_bytes_peak": 123805369856,
|
| 34 |
+
"allocated_bytes_current": 38987459072,
|
| 35 |
+
"allocated_bytes_peak": 123805369856,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 16318633472,
|
| 38 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 140393840640,
|
| 46 |
+
"reserved_bytes_peak": 140393840640,
|
| 47 |
+
"segment_count_current": 979,
|
| 48 |
+
"segment_count_peak": 979,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:49.098370Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16318633472,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 39258380288,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 140393840640,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 140393840640,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 979,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 979,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.691999Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:78de216bceed7727e49d5802db3c33c9402b572d8949a983c77023509c976944
|
| 3 |
+
size 356377
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38973895680,
|
| 3 |
+
"active_bytes_current_delta": 15925412352,
|
| 4 |
+
"active_bytes_peak": 109599972864,
|
| 5 |
+
"active_bytes_peak_delta": 86538906624,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38973895680,
|
| 8 |
+
"active_bytes_peak": 109599972864,
|
| 9 |
+
"allocated_bytes_current": 38973895680,
|
| 10 |
+
"allocated_bytes_peak": 109599972864,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 16290253824,
|
| 13 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 110899494912,
|
| 21 |
+
"reserved_bytes_peak": 110899494912,
|
| 22 |
+
"segment_count_current": 850,
|
| 23 |
+
"segment_count_peak": 850,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.258776Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38973895680,
|
| 28 |
+
"allocated_bytes_current_delta": 15925412352,
|
| 29 |
+
"allocated_bytes_peak": 109599972864,
|
| 30 |
+
"allocated_bytes_peak_delta": 86538906624,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23048483328,
|
| 33 |
+
"active_bytes_peak": 23061066240,
|
| 34 |
+
"allocated_bytes_current": 23048483328,
|
| 35 |
+
"allocated_bytes_peak": 23061066240,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 515116544,
|
| 38 |
+
"inactive_split_bytes_peak": 883057664,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 25784483840,
|
| 46 |
+
"reserved_bytes_peak": 25784483840,
|
| 47 |
+
"segment_count_current": 558,
|
| 48 |
+
"segment_count_peak": 558,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:17.758943Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16290253824,
|
| 55 |
+
"inactive_split_bytes_current_delta": 15775137280,
|
| 56 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 27610600448,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 110899494912,
|
| 68 |
+
"reserved_bytes_current_delta": 85115011072,
|
| 69 |
+
"reserved_bytes_peak": 110899494912,
|
| 70 |
+
"reserved_bytes_peak_delta": 85115011072,
|
| 71 |
+
"segment_count_current": 850,
|
| 72 |
+
"segment_count_current_delta": 292,
|
| 73 |
+
"segment_count_peak": 850,
|
| 74 |
+
"segment_count_peak_delta": 292,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.318988Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7091e3a842cdfd425213707c3f5267d552da2f9e66e04529c51e6eaf79d149de
|
| 3 |
+
size 3038267
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38973895680,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123793538560,
|
| 5 |
+
"active_bytes_peak_delta": 14193565696,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38973895680,
|
| 8 |
+
"active_bytes_peak": 123793538560,
|
| 9 |
+
"allocated_bytes_current": 38973895680,
|
| 10 |
+
"allocated_bytes_peak": 123793538560,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 16290253824,
|
| 13 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 127586533376,
|
| 21 |
+
"reserved_bytes_peak": 127586533376,
|
| 22 |
+
"segment_count_current": 861,
|
| 23 |
+
"segment_count_peak": 861,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:45.929094Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38973895680,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123793538560,
|
| 30 |
+
"allocated_bytes_peak_delta": 14193565696,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38973895680,
|
| 33 |
+
"active_bytes_peak": 109599972864,
|
| 34 |
+
"allocated_bytes_current": 38973895680,
|
| 35 |
+
"allocated_bytes_peak": 109599972864,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 16290253824,
|
| 38 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 110899494912,
|
| 46 |
+
"reserved_bytes_peak": 110899494912,
|
| 47 |
+
"segment_count_current": 850,
|
| 48 |
+
"segment_count_peak": 850,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.327786Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16290253824,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127586533376,
|
| 68 |
+
"reserved_bytes_current_delta": 16687038464,
|
| 69 |
+
"reserved_bytes_peak": 127586533376,
|
| 70 |
+
"reserved_bytes_peak_delta": 16687038464,
|
| 71 |
+
"segment_count_current": 861,
|
| 72 |
+
"segment_count_current_delta": 11,
|
| 73 |
+
"segment_count_peak": 861,
|
| 74 |
+
"segment_count_peak_delta": 11,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:45.981568Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:921ccb5f50ad1a49b8d0912c1cc8e07ff0bbfad73b6f251c7525b20aa860cf18
|
| 3 |
+
size 2983046
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38973895680,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123795095040,
|
| 5 |
+
"active_bytes_peak_delta": 1556480,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38973895680,
|
| 8 |
+
"active_bytes_peak": 123795095040,
|
| 9 |
+
"allocated_bytes_current": 38973895680,
|
| 10 |
+
"allocated_bytes_peak": 123795095040,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 16290253824,
|
| 13 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 127586533376,
|
| 21 |
+
"reserved_bytes_peak": 127586533376,
|
| 22 |
+
"segment_count_current": 861,
|
| 23 |
+
"segment_count_peak": 861,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:46.587246Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38973895680,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123795095040,
|
| 30 |
+
"allocated_bytes_peak_delta": 1556480,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38973895680,
|
| 33 |
+
"active_bytes_peak": 123793538560,
|
| 34 |
+
"allocated_bytes_current": 38973895680,
|
| 35 |
+
"allocated_bytes_peak": 123793538560,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 16290253824,
|
| 38 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 127586533376,
|
| 46 |
+
"reserved_bytes_peak": 127586533376,
|
| 47 |
+
"segment_count_current": 861,
|
| 48 |
+
"segment_count_peak": 861,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:45.987587Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 16290253824,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 127586533376,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 127586533376,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 861,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 861,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:46.639856Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9531abba1c87d4d6b1014745dfad6fdcfdb5457308bed0c61f600d1cc55ec5c4
|
| 3 |
+
size 2959464
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38991856640,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 123795095040,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38991856640,
|
| 8 |
+
"active_bytes_peak": 123795095040,
|
| 9 |
+
"allocated_bytes_current": 38991856640,
|
| 10 |
+
"allocated_bytes_peak": 123795095040,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 18444942336,
|
| 13 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 142524547072,
|
| 21 |
+
"reserved_bytes_peak": 142524547072,
|
| 22 |
+
"segment_count_current": 962,
|
| 23 |
+
"segment_count_peak": 962,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:48.442905Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38991856640,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 123795095040,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38973895680,
|
| 33 |
+
"active_bytes_peak": 123795095040,
|
| 34 |
+
"allocated_bytes_current": 38973895680,
|
| 35 |
+
"allocated_bytes_peak": 123795095040,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 16290253824,
|
| 38 |
+
"inactive_split_bytes_peak": 28493658112,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 127586533376,
|
| 46 |
+
"reserved_bytes_peak": 127586533376,
|
| 47 |
+
"segment_count_current": 861,
|
| 48 |
+
"segment_count_peak": 861,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:46.645936Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18444942336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2154688512,
|
| 56 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 12891031040,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142524547072,
|
| 68 |
+
"reserved_bytes_current_delta": 14938013696,
|
| 69 |
+
"reserved_bytes_peak": 142524547072,
|
| 70 |
+
"reserved_bytes_peak_delta": 14938013696,
|
| 71 |
+
"segment_count_current": 962,
|
| 72 |
+
"segment_count_current_delta": 101,
|
| 73 |
+
"segment_count_peak": 962,
|
| 74 |
+
"segment_count_peak_delta": 101,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:48.502282Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc766681d2a1724bc84a2bead9bfdc7b53d38c247c32707168acff7efc391325
|
| 3 |
+
size 3106732
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38991856640,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123795095040,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38991856640,
|
| 8 |
+
"active_bytes_peak": 123795095040,
|
| 9 |
+
"allocated_bytes_current": 38991856640,
|
| 10 |
+
"allocated_bytes_peak": 123795095040,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 18444942336,
|
| 13 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 142524547072,
|
| 21 |
+
"reserved_bytes_peak": 142524547072,
|
| 22 |
+
"segment_count_current": 962,
|
| 23 |
+
"segment_count_peak": 962,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.089185Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38991856640,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123795095040,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38991856640,
|
| 33 |
+
"active_bytes_peak": 123795095040,
|
| 34 |
+
"allocated_bytes_current": 38991856640,
|
| 35 |
+
"allocated_bytes_peak": 123795095040,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 18444942336,
|
| 38 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 142524547072,
|
| 46 |
+
"reserved_bytes_peak": 142524547072,
|
| 47 |
+
"segment_count_current": 962,
|
| 48 |
+
"segment_count_peak": 962,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:48.509296Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18444942336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142524547072,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142524547072,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 962,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 962,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.097168Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12265aafa532e83b466dd405387b287217c43b5f574287368cbd7661d56b9682
|
| 3 |
+
size 342163
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 38991856640,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123795095040,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 38991856640,
|
| 8 |
+
"active_bytes_peak": 123795095040,
|
| 9 |
+
"allocated_bytes_current": 38991856640,
|
| 10 |
+
"allocated_bytes_peak": 123795095040,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 18444942336,
|
| 13 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 142524547072,
|
| 21 |
+
"reserved_bytes_peak": 142524547072,
|
| 22 |
+
"segment_count_current": 962,
|
| 23 |
+
"segment_count_peak": 962,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:23:49.684043Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 38991856640,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123795095040,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 38991856640,
|
| 33 |
+
"active_bytes_peak": 123795095040,
|
| 34 |
+
"allocated_bytes_current": 38991856640,
|
| 35 |
+
"allocated_bytes_peak": 123795095040,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 18444942336,
|
| 38 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 142524547072,
|
| 46 |
+
"reserved_bytes_peak": 142524547072,
|
| 47 |
+
"segment_count_current": 962,
|
| 48 |
+
"segment_count_peak": 962,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:23:49.098209Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18444942336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 41384689152,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142524547072,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142524547072,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 962,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 962,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:23:49.692003Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0ecac16ad4203410bfbb2d55f36c425f98f921cc60078918955a8745939a1f7
|
| 3 |
+
size 342163
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=DSV2_LITE_L24
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-better-memory-dump-support
|
| 6 |
+
commit=c3a8a8a7
|
| 7 |
+
num_layers=24
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=10944
|
| 10 |
+
moe_ffn_hidden=1408
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=6
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=8650752
|
| 15 |
+
heads=16
|
| 16 |
+
kv_groups=0
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=2816
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=1
|
| 22 |
+
dense_first_layers=1
|
| 23 |
+
moe_layers=23
|
| 24 |
+
rotary_base=10000
|
| 25 |
+
rotary_scaling=40
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=seq_aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=49152
|
| 39 |
+
mean_tokens_per_expert=6144
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.25
|
| 44 |
+
cuda_graph=1
|
| 45 |
+
probe_mode=0
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=memory
|
| 48 |
+
memory_dump_mode=iteration
|
| 49 |
+
memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=6
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=768
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|