granularity results echo/tierbmem @ 2026-08-26T19:10:38Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env +61 -0
- echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log +0 -0
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42478680576,
|
| 3 |
+
"active_bytes_current_delta": 19430297088,
|
| 4 |
+
"active_bytes_peak": 113227885568,
|
| 5 |
+
"active_bytes_peak_delta": 90166919168,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42478680576,
|
| 8 |
+
"active_bytes_peak": 113227885568,
|
| 9 |
+
"allocated_bytes_current": 42478680576,
|
| 10 |
+
"allocated_bytes_peak": 113227885568,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 18456167936,
|
| 13 |
+
"inactive_split_bytes_peak": 30415799296,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 117411151872,
|
| 21 |
+
"reserved_bytes_peak": 117411151872,
|
| 22 |
+
"segment_count_current": 920,
|
| 23 |
+
"segment_count_peak": 920,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.080066Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42478680576,
|
| 28 |
+
"allocated_bytes_current_delta": 19430297088,
|
| 29 |
+
"allocated_bytes_peak": 113227885568,
|
| 30 |
+
"allocated_bytes_peak_delta": 90166919168,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23048383488,
|
| 33 |
+
"active_bytes_peak": 23060966400,
|
| 34 |
+
"allocated_bytes_current": 23048383488,
|
| 35 |
+
"allocated_bytes_peak": 23060966400,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 376804352,
|
| 38 |
+
"inactive_split_bytes_peak": 883235328,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 25623003136,
|
| 46 |
+
"reserved_bytes_peak": 25623003136,
|
| 47 |
+
"segment_count_current": 551,
|
| 48 |
+
"segment_count_peak": 551,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:03:23.745553Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18456167936,
|
| 55 |
+
"inactive_split_bytes_current_delta": 18079363584,
|
| 56 |
+
"inactive_split_bytes_peak": 30415799296,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 29532563968,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 117411151872,
|
| 68 |
+
"reserved_bytes_current_delta": 91788148736,
|
| 69 |
+
"reserved_bytes_peak": 117411151872,
|
| 70 |
+
"reserved_bytes_peak_delta": 91788148736,
|
| 71 |
+
"segment_count_current": 920,
|
| 72 |
+
"segment_count_current_delta": 369,
|
| 73 |
+
"segment_count_peak": 920,
|
| 74 |
+
"segment_count_peak_delta": 369,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.162509Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d83729cb37450461395f2b17a82bced8dc5205c1a1dfc7f01154f3a556b77380
|
| 3 |
+
size 4114465
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42478680576,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127729292800,
|
| 5 |
+
"active_bytes_peak_delta": 14501407232,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42478680576,
|
| 8 |
+
"active_bytes_peak": 127729292800,
|
| 9 |
+
"allocated_bytes_current": 42478680576,
|
| 10 |
+
"allocated_bytes_peak": 127452468736,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 18548442624,
|
| 13 |
+
"inactive_split_bytes_peak": 30684399104,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 129807417344,
|
| 21 |
+
"reserved_bytes_peak": 129807417344,
|
| 22 |
+
"segment_count_current": 931,
|
| 23 |
+
"segment_count_peak": 931,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.816236Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42478680576,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127452468736,
|
| 30 |
+
"allocated_bytes_peak_delta": 14224583168,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42478680576,
|
| 33 |
+
"active_bytes_peak": 113227885568,
|
| 34 |
+
"allocated_bytes_current": 42478680576,
|
| 35 |
+
"allocated_bytes_peak": 113227885568,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 18456167936,
|
| 38 |
+
"inactive_split_bytes_peak": 30415799296,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 117411151872,
|
| 46 |
+
"reserved_bytes_peak": 117411151872,
|
| 47 |
+
"segment_count_current": 920,
|
| 48 |
+
"segment_count_peak": 920,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.173774Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18548442624,
|
| 55 |
+
"inactive_split_bytes_current_delta": 92274688,
|
| 56 |
+
"inactive_split_bytes_peak": 30684399104,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 268599808,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 129807417344,
|
| 68 |
+
"reserved_bytes_current_delta": 12396265472,
|
| 69 |
+
"reserved_bytes_peak": 129807417344,
|
| 70 |
+
"reserved_bytes_peak_delta": 12396265472,
|
| 71 |
+
"segment_count_current": 931,
|
| 72 |
+
"segment_count_current_delta": 11,
|
| 73 |
+
"segment_count_peak": 931,
|
| 74 |
+
"segment_count_peak_delta": 11,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.891028Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ebe4fbadd82e9e91e33d1c17ef2c8d74cfd0adc06e63be0dc3e564db86ac3b95
|
| 3 |
+
size 4503524
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42478680576,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127731652096,
|
| 5 |
+
"active_bytes_peak_delta": 2359296,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42478680576,
|
| 8 |
+
"active_bytes_peak": 127731652096,
|
| 9 |
+
"allocated_bytes_current": 42478680576,
|
| 10 |
+
"allocated_bytes_peak": 127454828032,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 18458265088,
|
| 13 |
+
"inactive_split_bytes_peak": 30684399104,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 129807417344,
|
| 21 |
+
"reserved_bytes_peak": 129807417344,
|
| 22 |
+
"segment_count_current": 931,
|
| 23 |
+
"segment_count_peak": 931,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:03.544793Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42478680576,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127454828032,
|
| 30 |
+
"allocated_bytes_peak_delta": 2359296,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42478680576,
|
| 33 |
+
"active_bytes_peak": 127729292800,
|
| 34 |
+
"allocated_bytes_current": 42478680576,
|
| 35 |
+
"allocated_bytes_peak": 127452468736,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 18548442624,
|
| 38 |
+
"inactive_split_bytes_peak": 30684399104,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 129807417344,
|
| 46 |
+
"reserved_bytes_peak": 129807417344,
|
| 47 |
+
"segment_count_current": 931,
|
| 48 |
+
"segment_count_peak": 931,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.899770Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18458265088,
|
| 55 |
+
"inactive_split_bytes_current_delta": -90177536,
|
| 56 |
+
"inactive_split_bytes_peak": 30684399104,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 129807417344,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 129807417344,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 931,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 931,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:03.620923Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fac8cfeefd3e11c9761bb1030a4b904986da1d50ce422eb6127c83e2cd002336
|
| 3 |
+
size 4520747
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42496641536,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 130557001728,
|
| 5 |
+
"active_bytes_peak_delta": 2825349632,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42496641536,
|
| 8 |
+
"active_bytes_peak": 130557001728,
|
| 9 |
+
"allocated_bytes_current": 42496641536,
|
| 10 |
+
"allocated_bytes_peak": 127454828032,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 20610856448,
|
| 13 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 155212316672,
|
| 21 |
+
"reserved_bytes_peak": 155212316672,
|
| 22 |
+
"segment_count_current": 1130,
|
| 23 |
+
"segment_count_peak": 1130,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:05.730047Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42496641536,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 127454828032,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42478680576,
|
| 33 |
+
"active_bytes_peak": 127731652096,
|
| 34 |
+
"allocated_bytes_current": 42478680576,
|
| 35 |
+
"allocated_bytes_peak": 127454828032,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 18458265088,
|
| 38 |
+
"inactive_split_bytes_peak": 30684399104,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 129807417344,
|
| 46 |
+
"reserved_bytes_peak": 129807417344,
|
| 47 |
+
"segment_count_current": 931,
|
| 48 |
+
"segment_count_peak": 931,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:03.629839Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20610856448,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2152591360,
|
| 56 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 15975260160,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155212316672,
|
| 68 |
+
"reserved_bytes_current_delta": 25404899328,
|
| 69 |
+
"reserved_bytes_peak": 155212316672,
|
| 70 |
+
"reserved_bytes_peak_delta": 25404899328,
|
| 71 |
+
"segment_count_current": 1130,
|
| 72 |
+
"segment_count_current_delta": 199,
|
| 73 |
+
"segment_count_peak": 1130,
|
| 74 |
+
"segment_count_peak_delta": 199,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:05.818896Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:10ac130196532840e49750a16588a05035df2819076786300a6423ec1be2210e
|
| 3 |
+
size 4737184
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42496641536,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130557001728,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42496641536,
|
| 8 |
+
"active_bytes_peak": 130557001728,
|
| 9 |
+
"allocated_bytes_current": 42496641536,
|
| 10 |
+
"allocated_bytes_peak": 127454828032,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 20610856448,
|
| 13 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 155212316672,
|
| 21 |
+
"reserved_bytes_peak": 155212316672,
|
| 22 |
+
"segment_count_current": 1130,
|
| 23 |
+
"segment_count_peak": 1130,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:06.418698Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42496641536,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127454828032,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42496641536,
|
| 33 |
+
"active_bytes_peak": 130557001728,
|
| 34 |
+
"allocated_bytes_current": 42496641536,
|
| 35 |
+
"allocated_bytes_peak": 127454828032,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 20610856448,
|
| 38 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 155212316672,
|
| 46 |
+
"reserved_bytes_peak": 155212316672,
|
| 47 |
+
"segment_count_current": 1130,
|
| 48 |
+
"segment_count_peak": 1130,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:05.828949Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20610856448,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155212316672,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155212316672,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1130,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1130,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:06.429348Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:18b40bf0480d107b9ef1624b9ce4794741d1aa07c358c1fc50687acd34ec779e
|
| 3 |
+
size 381106
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42496641536,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130557001728,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42496641536,
|
| 8 |
+
"active_bytes_peak": 130557001728,
|
| 9 |
+
"allocated_bytes_current": 42496641536,
|
| 10 |
+
"allocated_bytes_peak": 127454828032,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 20610856448,
|
| 13 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 155212316672,
|
| 21 |
+
"reserved_bytes_peak": 155212316672,
|
| 22 |
+
"segment_count_current": 1130,
|
| 23 |
+
"segment_count_peak": 1130,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:07.024122Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42496641536,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127454828032,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42496641536,
|
| 33 |
+
"active_bytes_peak": 130557001728,
|
| 34 |
+
"allocated_bytes_current": 42496641536,
|
| 35 |
+
"allocated_bytes_peak": 127454828032,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 20610856448,
|
| 38 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 155212316672,
|
| 46 |
+
"reserved_bytes_peak": 155212316672,
|
| 47 |
+
"segment_count_current": 1130,
|
| 48 |
+
"segment_count_peak": 1130,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:06.430441Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20610856448,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 46659659264,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155212316672,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155212316672,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1130,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1130,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:07.034324Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9c81cf1f8019b801f8aa391c3519ce153b5a72c6ec7200125305317d1c965999
|
| 3 |
+
size 381106
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42487518720,
|
| 3 |
+
"active_bytes_current_delta": 19434335744,
|
| 4 |
+
"active_bytes_peak": 113240445952,
|
| 5 |
+
"active_bytes_peak_delta": 90174680064,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42487518720,
|
| 8 |
+
"active_bytes_peak": 113240445952,
|
| 9 |
+
"allocated_bytes_current": 42487518720,
|
| 10 |
+
"allocated_bytes_peak": 113240445952,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 18376026624,
|
| 13 |
+
"inactive_split_bytes_peak": 30425401344,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 117409054720,
|
| 21 |
+
"reserved_bytes_peak": 117409054720,
|
| 22 |
+
"segment_count_current": 919,
|
| 23 |
+
"segment_count_peak": 919,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.080015Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42487518720,
|
| 28 |
+
"allocated_bytes_current_delta": 19434335744,
|
| 29 |
+
"allocated_bytes_peak": 113240445952,
|
| 30 |
+
"allocated_bytes_peak_delta": 90174680064,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23053182976,
|
| 33 |
+
"active_bytes_peak": 23065765888,
|
| 34 |
+
"allocated_bytes_current": 23053182976,
|
| 35 |
+
"allocated_bytes_peak": 23065765888,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 462182400,
|
| 38 |
+
"inactive_split_bytes_peak": 883235328,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 25623003136,
|
| 46 |
+
"reserved_bytes_peak": 25623003136,
|
| 47 |
+
"segment_count_current": 551,
|
| 48 |
+
"segment_count_peak": 551,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:03:23.745648Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18376026624,
|
| 55 |
+
"inactive_split_bytes_current_delta": 17913844224,
|
| 56 |
+
"inactive_split_bytes_peak": 30425401344,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 29542166016,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 117409054720,
|
| 68 |
+
"reserved_bytes_current_delta": 91786051584,
|
| 69 |
+
"reserved_bytes_peak": 117409054720,
|
| 70 |
+
"reserved_bytes_peak_delta": 91786051584,
|
| 71 |
+
"segment_count_current": 919,
|
| 72 |
+
"segment_count_current_delta": 368,
|
| 73 |
+
"segment_count_peak": 919,
|
| 74 |
+
"segment_count_peak_delta": 368,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.166044Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:56b903ccb299ef35670c726458cd25437f737f6bf7790813de00992262510475
|
| 3 |
+
size 4134331
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42487518720,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127585480192,
|
| 5 |
+
"active_bytes_peak_delta": 14345034240,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42487518720,
|
| 8 |
+
"active_bytes_peak": 127585480192,
|
| 9 |
+
"allocated_bytes_current": 42487518720,
|
| 10 |
+
"allocated_bytes_peak": 127447068160,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 18468301312,
|
| 13 |
+
"inactive_split_bytes_peak": 30425401344,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 131946512384,
|
| 21 |
+
"reserved_bytes_peak": 131946512384,
|
| 22 |
+
"segment_count_current": 928,
|
| 23 |
+
"segment_count_peak": 928,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.816170Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42487518720,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127447068160,
|
| 30 |
+
"allocated_bytes_peak_delta": 14206622208,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42487518720,
|
| 33 |
+
"active_bytes_peak": 113240445952,
|
| 34 |
+
"allocated_bytes_current": 42487518720,
|
| 35 |
+
"allocated_bytes_peak": 113240445952,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 18376026624,
|
| 38 |
+
"inactive_split_bytes_peak": 30425401344,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 117409054720,
|
| 46 |
+
"reserved_bytes_peak": 117409054720,
|
| 47 |
+
"segment_count_current": 919,
|
| 48 |
+
"segment_count_peak": 919,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.178637Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18468301312,
|
| 55 |
+
"inactive_split_bytes_current_delta": 92274688,
|
| 56 |
+
"inactive_split_bytes_peak": 30425401344,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131946512384,
|
| 68 |
+
"reserved_bytes_current_delta": 14537457664,
|
| 69 |
+
"reserved_bytes_peak": 131946512384,
|
| 70 |
+
"reserved_bytes_peak_delta": 14537457664,
|
| 71 |
+
"segment_count_current": 928,
|
| 72 |
+
"segment_count_current_delta": 9,
|
| 73 |
+
"segment_count_peak": 928,
|
| 74 |
+
"segment_count_peak_delta": 9,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.892646Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:96de714f4a3d9f18d4e49013712ee500a525c925beb064029d5c21ddadb9cbe1
|
| 3 |
+
size 4529243
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42487518720,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127718985216,
|
| 5 |
+
"active_bytes_peak_delta": 133505024,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42487518720,
|
| 8 |
+
"active_bytes_peak": 127718985216,
|
| 9 |
+
"allocated_bytes_current": 42487518720,
|
| 10 |
+
"allocated_bytes_peak": 127447068160,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 18470398464,
|
| 13 |
+
"inactive_split_bytes_peak": 31822153216,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 131946512384,
|
| 21 |
+
"reserved_bytes_peak": 131946512384,
|
| 22 |
+
"segment_count_current": 928,
|
| 23 |
+
"segment_count_peak": 928,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:03.544740Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42487518720,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127447068160,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42487518720,
|
| 33 |
+
"active_bytes_peak": 127585480192,
|
| 34 |
+
"allocated_bytes_current": 42487518720,
|
| 35 |
+
"allocated_bytes_peak": 127447068160,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 18468301312,
|
| 38 |
+
"inactive_split_bytes_peak": 30425401344,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 131946512384,
|
| 46 |
+
"reserved_bytes_peak": 131946512384,
|
| 47 |
+
"segment_count_current": 928,
|
| 48 |
+
"segment_count_peak": 928,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.902746Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18470398464,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 31822153216,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1396751872,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131946512384,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 131946512384,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 928,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 928,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:03.619957Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c8afa962f2fe520b956cebd932c0a11446c25f3b7e7950d89aa43ec0c43724a6
|
| 3 |
+
size 4545972
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42505479680,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 130565839872,
|
| 5 |
+
"active_bytes_peak_delta": 2846854656,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42505479680,
|
| 8 |
+
"active_bytes_peak": 130565839872,
|
| 9 |
+
"allocated_bytes_current": 42505479680,
|
| 10 |
+
"allocated_bytes_peak": 127447068160,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 20530715136,
|
| 13 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 155233288192,
|
| 21 |
+
"reserved_bytes_peak": 155233288192,
|
| 22 |
+
"segment_count_current": 1097,
|
| 23 |
+
"segment_count_peak": 1097,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:05.729868Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42505479680,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 127447068160,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42487518720,
|
| 33 |
+
"active_bytes_peak": 127718985216,
|
| 34 |
+
"allocated_bytes_current": 42487518720,
|
| 35 |
+
"allocated_bytes_peak": 127447068160,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 18470398464,
|
| 38 |
+
"inactive_split_bytes_peak": 31822153216,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 131946512384,
|
| 46 |
+
"reserved_bytes_peak": 131946512384,
|
| 47 |
+
"segment_count_current": 928,
|
| 48 |
+
"segment_count_peak": 928,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:03.629418Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20530715136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2060316672,
|
| 56 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 14847542272,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155233288192,
|
| 68 |
+
"reserved_bytes_current_delta": 23286775808,
|
| 69 |
+
"reserved_bytes_peak": 155233288192,
|
| 70 |
+
"reserved_bytes_peak_delta": 23286775808,
|
| 71 |
+
"segment_count_current": 1097,
|
| 72 |
+
"segment_count_current_delta": 169,
|
| 73 |
+
"segment_count_peak": 1097,
|
| 74 |
+
"segment_count_peak_delta": 169,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:05.816882Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:624ccf54acb5bc1133e019f5644605beeb1cdb228b6a43a6a0670036ce80281d
|
| 3 |
+
size 4761905
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42505479680,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130565839872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42505479680,
|
| 8 |
+
"active_bytes_peak": 130565839872,
|
| 9 |
+
"allocated_bytes_current": 42505479680,
|
| 10 |
+
"allocated_bytes_peak": 127447068160,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 20530715136,
|
| 13 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 155233288192,
|
| 21 |
+
"reserved_bytes_peak": 155233288192,
|
| 22 |
+
"segment_count_current": 1097,
|
| 23 |
+
"segment_count_peak": 1097,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:06.418658Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42505479680,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127447068160,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42505479680,
|
| 33 |
+
"active_bytes_peak": 130565839872,
|
| 34 |
+
"allocated_bytes_current": 42505479680,
|
| 35 |
+
"allocated_bytes_peak": 127447068160,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 20530715136,
|
| 38 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 155233288192,
|
| 46 |
+
"reserved_bytes_peak": 155233288192,
|
| 47 |
+
"segment_count_current": 1097,
|
| 48 |
+
"segment_count_peak": 1097,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:05.829885Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20530715136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155233288192,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155233288192,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1097,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1097,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:06.429246Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:01890f6d738e8cc73fafbc7c5b378623fb52a98b01996eeb5a9055b6de5cf06f
|
| 3 |
+
size 403620
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42505479680,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130565839872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42505479680,
|
| 8 |
+
"active_bytes_peak": 130565839872,
|
| 9 |
+
"allocated_bytes_current": 42505479680,
|
| 10 |
+
"allocated_bytes_peak": 127447068160,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 20530715136,
|
| 13 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 155233288192,
|
| 21 |
+
"reserved_bytes_peak": 155233288192,
|
| 22 |
+
"segment_count_current": 1097,
|
| 23 |
+
"segment_count_peak": 1097,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:07.024063Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42505479680,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127447068160,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42505479680,
|
| 33 |
+
"active_bytes_peak": 130565839872,
|
| 34 |
+
"allocated_bytes_current": 42505479680,
|
| 35 |
+
"allocated_bytes_peak": 127447068160,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 20530715136,
|
| 38 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 155233288192,
|
| 46 |
+
"reserved_bytes_peak": 155233288192,
|
| 47 |
+
"segment_count_current": 1097,
|
| 48 |
+
"segment_count_peak": 1097,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:06.430320Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20530715136,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 46669695488,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155233288192,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155233288192,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1097,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1097,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:07.033441Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:28828d5ebd6237774c5b03e0400603dab08621171960bff1a06def72d36e5682
|
| 3 |
+
size 403620
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42485283328,
|
| 3 |
+
"active_bytes_current_delta": 19439536128,
|
| 4 |
+
"active_bytes_peak": 113234178048,
|
| 5 |
+
"active_bytes_peak_delta": 90175847936,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42485283328,
|
| 8 |
+
"active_bytes_peak": 113234178048,
|
| 9 |
+
"allocated_bytes_current": 42485283328,
|
| 10 |
+
"allocated_bytes_peak": 113234178048,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 18608948736,
|
| 13 |
+
"inactive_split_bytes_peak": 30468116992,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 117549563904,
|
| 21 |
+
"reserved_bytes_peak": 117549563904,
|
| 22 |
+
"segment_count_current": 926,
|
| 23 |
+
"segment_count_peak": 926,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.079986Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42485283328,
|
| 28 |
+
"allocated_bytes_current_delta": 19439536128,
|
| 29 |
+
"allocated_bytes_peak": 113234178048,
|
| 30 |
+
"allocated_bytes_peak_delta": 90175847936,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23045747200,
|
| 33 |
+
"active_bytes_peak": 23058330112,
|
| 34 |
+
"allocated_bytes_current": 23045747200,
|
| 35 |
+
"allocated_bytes_peak": 23058330112,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 358469120,
|
| 38 |
+
"inactive_split_bytes_peak": 883235328,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 25761415168,
|
| 46 |
+
"reserved_bytes_peak": 25761415168,
|
| 47 |
+
"segment_count_current": 557,
|
| 48 |
+
"segment_count_peak": 557,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:03:23.745539Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18608948736,
|
| 55 |
+
"inactive_split_bytes_current_delta": 18250479616,
|
| 56 |
+
"inactive_split_bytes_peak": 30468116992,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 29584881664,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 117549563904,
|
| 68 |
+
"reserved_bytes_current_delta": 91788148736,
|
| 69 |
+
"reserved_bytes_peak": 117549563904,
|
| 70 |
+
"reserved_bytes_peak_delta": 91788148736,
|
| 71 |
+
"segment_count_current": 926,
|
| 72 |
+
"segment_count_current_delta": 369,
|
| 73 |
+
"segment_count_peak": 926,
|
| 74 |
+
"segment_count_peak_delta": 369,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.162223Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e5087269156536422fe460c092ada4e54ccaa027b0479bb29957631815242864
|
| 3 |
+
size 4119126
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42485283328,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127602459136,
|
| 5 |
+
"active_bytes_peak_delta": 14368281088,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42485283328,
|
| 8 |
+
"active_bytes_peak": 127602459136,
|
| 9 |
+
"allocated_bytes_current": 42485283328,
|
| 10 |
+
"allocated_bytes_peak": 127464047104,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 18701223424,
|
| 13 |
+
"inactive_split_bytes_peak": 30468116992,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 132095410176,
|
| 21 |
+
"reserved_bytes_peak": 132095410176,
|
| 22 |
+
"segment_count_current": 939,
|
| 23 |
+
"segment_count_peak": 939,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.816227Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42485283328,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127464047104,
|
| 30 |
+
"allocated_bytes_peak_delta": 14229869056,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42485283328,
|
| 33 |
+
"active_bytes_peak": 113234178048,
|
| 34 |
+
"allocated_bytes_current": 42485283328,
|
| 35 |
+
"allocated_bytes_peak": 113234178048,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 18608948736,
|
| 38 |
+
"inactive_split_bytes_peak": 30468116992,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 117549563904,
|
| 46 |
+
"reserved_bytes_peak": 117549563904,
|
| 47 |
+
"segment_count_current": 926,
|
| 48 |
+
"segment_count_peak": 926,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.174428Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18701223424,
|
| 55 |
+
"inactive_split_bytes_current_delta": 92274688,
|
| 56 |
+
"inactive_split_bytes_peak": 30468116992,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132095410176,
|
| 68 |
+
"reserved_bytes_current_delta": 14545846272,
|
| 69 |
+
"reserved_bytes_peak": 132095410176,
|
| 70 |
+
"reserved_bytes_peak_delta": 14545846272,
|
| 71 |
+
"segment_count_current": 939,
|
| 72 |
+
"segment_count_current_delta": 13,
|
| 73 |
+
"segment_count_peak": 939,
|
| 74 |
+
"segment_count_peak_delta": 13,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.893379Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1a6f365f5a7ce3398fb05d1c6cf8ae6aa3ecdc3210fe7c03d685d4a34d72590b
|
| 3 |
+
size 4509200
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42485283328,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127741936128,
|
| 5 |
+
"active_bytes_peak_delta": 139476992,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42485283328,
|
| 8 |
+
"active_bytes_peak": 127741936128,
|
| 9 |
+
"allocated_bytes_current": 42485283328,
|
| 10 |
+
"allocated_bytes_peak": 127465112064,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 18703320576,
|
| 13 |
+
"inactive_split_bytes_peak": 32325633536,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 132187684864,
|
| 21 |
+
"reserved_bytes_peak": 132187684864,
|
| 22 |
+
"segment_count_current": 940,
|
| 23 |
+
"segment_count_peak": 940,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:03.544749Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42485283328,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127465112064,
|
| 30 |
+
"allocated_bytes_peak_delta": 1064960,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42485283328,
|
| 33 |
+
"active_bytes_peak": 127602459136,
|
| 34 |
+
"allocated_bytes_current": 42485283328,
|
| 35 |
+
"allocated_bytes_peak": 127464047104,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 18701223424,
|
| 38 |
+
"inactive_split_bytes_peak": 30468116992,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 132095410176,
|
| 46 |
+
"reserved_bytes_peak": 132095410176,
|
| 47 |
+
"segment_count_current": 939,
|
| 48 |
+
"segment_count_peak": 939,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.903317Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18703320576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 32325633536,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1857516544,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132187684864,
|
| 68 |
+
"reserved_bytes_current_delta": 92274688,
|
| 69 |
+
"reserved_bytes_peak": 132187684864,
|
| 70 |
+
"reserved_bytes_peak_delta": 92274688,
|
| 71 |
+
"segment_count_current": 940,
|
| 72 |
+
"segment_count_current_delta": 1,
|
| 73 |
+
"segment_count_peak": 940,
|
| 74 |
+
"segment_count_peak_delta": 1,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:03.620155Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aa49457f319d10134dd62b75318b0f7766bf65ef9e67578bb76f120c9311583e
|
| 3 |
+
size 4526558
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42503244288,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 130563604480,
|
| 5 |
+
"active_bytes_peak_delta": 2821668352,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42503244288,
|
| 8 |
+
"active_bytes_peak": 130563604480,
|
| 9 |
+
"allocated_bytes_current": 42503244288,
|
| 10 |
+
"allocated_bytes_peak": 127465112064,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 20763637248,
|
| 13 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 155463974912,
|
| 21 |
+
"reserved_bytes_peak": 155463974912,
|
| 22 |
+
"segment_count_current": 1147,
|
| 23 |
+
"segment_count_peak": 1147,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:05.729858Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42503244288,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 127465112064,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42485283328,
|
| 33 |
+
"active_bytes_peak": 127741936128,
|
| 34 |
+
"allocated_bytes_current": 42485283328,
|
| 35 |
+
"allocated_bytes_peak": 127465112064,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 18703320576,
|
| 38 |
+
"inactive_split_bytes_peak": 32325633536,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 132187684864,
|
| 46 |
+
"reserved_bytes_peak": 132187684864,
|
| 47 |
+
"segment_count_current": 940,
|
| 48 |
+
"segment_count_peak": 940,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:03.629844Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20763637248,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2060316672,
|
| 56 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 14669258752,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155463974912,
|
| 68 |
+
"reserved_bytes_current_delta": 23276290048,
|
| 69 |
+
"reserved_bytes_peak": 155463974912,
|
| 70 |
+
"reserved_bytes_peak_delta": 23276290048,
|
| 71 |
+
"segment_count_current": 1147,
|
| 72 |
+
"segment_count_current_delta": 207,
|
| 73 |
+
"segment_count_peak": 1147,
|
| 74 |
+
"segment_count_peak_delta": 207,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:05.816618Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a24fc4e9476ffa47c1f1f2f9e278b6aa303a80de5d839eda5cb01d2d47d409c
|
| 3 |
+
size 4743045
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42503244288,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130563604480,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42503244288,
|
| 8 |
+
"active_bytes_peak": 130563604480,
|
| 9 |
+
"allocated_bytes_current": 42503244288,
|
| 10 |
+
"allocated_bytes_peak": 127465112064,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 20763637248,
|
| 13 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 155463974912,
|
| 21 |
+
"reserved_bytes_peak": 155463974912,
|
| 22 |
+
"segment_count_current": 1147,
|
| 23 |
+
"segment_count_peak": 1147,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:06.418670Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42503244288,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127465112064,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42503244288,
|
| 33 |
+
"active_bytes_peak": 130563604480,
|
| 34 |
+
"allocated_bytes_current": 42503244288,
|
| 35 |
+
"allocated_bytes_peak": 127465112064,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 20763637248,
|
| 38 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 155463974912,
|
| 46 |
+
"reserved_bytes_peak": 155463974912,
|
| 47 |
+
"segment_count_current": 1147,
|
| 48 |
+
"segment_count_peak": 1147,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:05.827858Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20763637248,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155463974912,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155463974912,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1147,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1147,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:06.429585Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aebbfe8d86a9a148c7944cf029e67444944214d147c175352f0f429eb5660d46
|
| 3 |
+
size 384967
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42503244288,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130563604480,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42503244288,
|
| 8 |
+
"active_bytes_peak": 130563604480,
|
| 9 |
+
"allocated_bytes_current": 42503244288,
|
| 10 |
+
"allocated_bytes_peak": 127465112064,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 20763637248,
|
| 13 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 155463974912,
|
| 21 |
+
"reserved_bytes_peak": 155463974912,
|
| 22 |
+
"segment_count_current": 1147,
|
| 23 |
+
"segment_count_peak": 1147,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:07.024063Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42503244288,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127465112064,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42503244288,
|
| 33 |
+
"active_bytes_peak": 130563604480,
|
| 34 |
+
"allocated_bytes_current": 42503244288,
|
| 35 |
+
"allocated_bytes_peak": 127465112064,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 20763637248,
|
| 38 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 155463974912,
|
| 46 |
+
"reserved_bytes_peak": 155463974912,
|
| 47 |
+
"segment_count_current": 1147,
|
| 48 |
+
"segment_count_peak": 1147,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:06.430686Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20763637248,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 46994892288,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155463974912,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155463974912,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1147,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1147,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:07.033945Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fd7f049c76c68f9bcc677e416ff783ce76c3ac862ca0efeb7dc803661ca50f8d
|
| 3 |
+
size 384967
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42481551360,
|
| 3 |
+
"active_bytes_current_delta": 19433245696,
|
| 4 |
+
"active_bytes_peak": 113246329856,
|
| 5 |
+
"active_bytes_peak_delta": 90185441280,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42481551360,
|
| 8 |
+
"active_bytes_peak": 113246329856,
|
| 9 |
+
"allocated_bytes_current": 42481551360,
|
| 10 |
+
"allocated_bytes_peak": 113246329856,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 18811910144,
|
| 13 |
+
"inactive_split_bytes_peak": 30718459904,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 117838970880,
|
| 21 |
+
"reserved_bytes_peak": 117838970880,
|
| 22 |
+
"segment_count_current": 927,
|
| 23 |
+
"segment_count_peak": 927,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.080054Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42481551360,
|
| 28 |
+
"allocated_bytes_current_delta": 19433245696,
|
| 29 |
+
"allocated_bytes_peak": 113246329856,
|
| 30 |
+
"allocated_bytes_peak_delta": 90185441280,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 23048305664,
|
| 33 |
+
"active_bytes_peak": 23060888576,
|
| 34 |
+
"allocated_bytes_current": 23048305664,
|
| 35 |
+
"allocated_bytes_peak": 23060888576,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 502711296,
|
| 38 |
+
"inactive_split_bytes_peak": 883235328,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 25784483840,
|
| 46 |
+
"reserved_bytes_peak": 25784483840,
|
| 47 |
+
"segment_count_current": 558,
|
| 48 |
+
"segment_count_peak": 558,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:03:23.745600Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18811910144,
|
| 55 |
+
"inactive_split_bytes_current_delta": 18309198848,
|
| 56 |
+
"inactive_split_bytes_peak": 30718459904,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 29835224576,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 117838970880,
|
| 68 |
+
"reserved_bytes_current_delta": 92054487040,
|
| 69 |
+
"reserved_bytes_peak": 117838970880,
|
| 70 |
+
"reserved_bytes_peak_delta": 92054487040,
|
| 71 |
+
"segment_count_current": 927,
|
| 72 |
+
"segment_count_current_delta": 369,
|
| 73 |
+
"segment_count_peak": 927,
|
| 74 |
+
"segment_count_peak_delta": 369,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.163072Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a173ef4ffb81f29a415ae4991c1534810f7f11a29159ed7ba8526ced8aed8e27
|
| 3 |
+
size 4131448
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42481551360,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127725126656,
|
| 5 |
+
"active_bytes_peak_delta": 14478796800,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42481551360,
|
| 8 |
+
"active_bytes_peak": 127725126656,
|
| 9 |
+
"allocated_bytes_current": 42481551360,
|
| 10 |
+
"allocated_bytes_peak": 127448441856,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 18906281984,
|
| 13 |
+
"inactive_split_bytes_peak": 31017596928,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 130228944896,
|
| 21 |
+
"reserved_bytes_peak": 130228944896,
|
| 22 |
+
"segment_count_current": 935,
|
| 23 |
+
"segment_count_peak": 935,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:02.816197Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42481551360,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127448441856,
|
| 30 |
+
"allocated_bytes_peak_delta": 14202112000,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42481551360,
|
| 33 |
+
"active_bytes_peak": 113246329856,
|
| 34 |
+
"allocated_bytes_current": 42481551360,
|
| 35 |
+
"allocated_bytes_peak": 113246329856,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 18811910144,
|
| 38 |
+
"inactive_split_bytes_peak": 30718459904,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 117838970880,
|
| 46 |
+
"reserved_bytes_peak": 117838970880,
|
| 47 |
+
"segment_count_current": 927,
|
| 48 |
+
"segment_count_peak": 927,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.175428Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18906281984,
|
| 55 |
+
"inactive_split_bytes_current_delta": 94371840,
|
| 56 |
+
"inactive_split_bytes_peak": 31017596928,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 299137024,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 130228944896,
|
| 68 |
+
"reserved_bytes_current_delta": 12389974016,
|
| 69 |
+
"reserved_bytes_peak": 130228944896,
|
| 70 |
+
"reserved_bytes_peak_delta": 12389974016,
|
| 71 |
+
"segment_count_current": 935,
|
| 72 |
+
"segment_count_current_delta": 8,
|
| 73 |
+
"segment_count_peak": 935,
|
| 74 |
+
"segment_count_peak_delta": 8,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:02.893988Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fd0e6496b8c4f5e5a893295cce203c7b318f3b559a9b814981578f0119c3b48a
|
| 3 |
+
size 4526765
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42481551360,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 127725126656,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42481551360,
|
| 8 |
+
"active_bytes_peak": 127725126656,
|
| 9 |
+
"allocated_bytes_current": 42481551360,
|
| 10 |
+
"allocated_bytes_peak": 127448441856,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 18904184832,
|
| 13 |
+
"inactive_split_bytes_peak": 31017596928,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 130321219584,
|
| 21 |
+
"reserved_bytes_peak": 130321219584,
|
| 22 |
+
"segment_count_current": 936,
|
| 23 |
+
"segment_count_peak": 936,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:03.544788Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42481551360,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127448441856,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42481551360,
|
| 33 |
+
"active_bytes_peak": 127725126656,
|
| 34 |
+
"allocated_bytes_current": 42481551360,
|
| 35 |
+
"allocated_bytes_peak": 127448441856,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 18906281984,
|
| 38 |
+
"inactive_split_bytes_peak": 31017596928,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 130228944896,
|
| 46 |
+
"reserved_bytes_peak": 130228944896,
|
| 47 |
+
"segment_count_current": 935,
|
| 48 |
+
"segment_count_peak": 935,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:02.904143Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 18904184832,
|
| 55 |
+
"inactive_split_bytes_current_delta": -2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 31017596928,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 130321219584,
|
| 68 |
+
"reserved_bytes_current_delta": 92274688,
|
| 69 |
+
"reserved_bytes_peak": 130321219584,
|
| 70 |
+
"reserved_bytes_peak_delta": 92274688,
|
| 71 |
+
"segment_count_current": 936,
|
| 72 |
+
"segment_count_current_delta": 1,
|
| 73 |
+
"segment_count_peak": 936,
|
| 74 |
+
"segment_count_peak_delta": 1,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:03.621671Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b186e0e3517c8a4ba9a3e4dca7e14ae4fc1117cc85b8233f13e6536b0f26ff3c
|
| 3 |
+
size 4545022
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42499512320,
|
| 3 |
+
"active_bytes_current_delta": 17960960,
|
| 4 |
+
"active_bytes_peak": 130559872512,
|
| 5 |
+
"active_bytes_peak_delta": 2834745856,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42499512320,
|
| 8 |
+
"active_bytes_peak": 130559872512,
|
| 9 |
+
"allocated_bytes_current": 42499512320,
|
| 10 |
+
"allocated_bytes_peak": 127448441856,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 20966598656,
|
| 13 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 155661107200,
|
| 21 |
+
"reserved_bytes_peak": 155661107200,
|
| 22 |
+
"segment_count_current": 1104,
|
| 23 |
+
"segment_count_peak": 1104,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:05.729875Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42499512320,
|
| 28 |
+
"allocated_bytes_current_delta": 17960960,
|
| 29 |
+
"allocated_bytes_peak": 127448441856,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42481551360,
|
| 33 |
+
"active_bytes_peak": 127725126656,
|
| 34 |
+
"allocated_bytes_current": 42481551360,
|
| 35 |
+
"allocated_bytes_peak": 127448441856,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 18904184832,
|
| 38 |
+
"inactive_split_bytes_peak": 31017596928,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 130321219584,
|
| 46 |
+
"reserved_bytes_peak": 130321219584,
|
| 47 |
+
"segment_count_current": 936,
|
| 48 |
+
"segment_count_peak": 936,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:03.631572Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20966598656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2062413824,
|
| 56 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 16180256768,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155661107200,
|
| 68 |
+
"reserved_bytes_current_delta": 25339887616,
|
| 69 |
+
"reserved_bytes_peak": 155661107200,
|
| 70 |
+
"reserved_bytes_peak_delta": 25339887616,
|
| 71 |
+
"segment_count_current": 1104,
|
| 72 |
+
"segment_count_current_delta": 168,
|
| 73 |
+
"segment_count_peak": 1104,
|
| 74 |
+
"segment_count_peak_delta": 168,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:05.818717Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e2a654dca35dfc9651a940d33d0c4f97e84e261cec94d4caed1e3bfe091e040
|
| 3 |
+
size 4761004
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42499512320,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130559872512,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42499512320,
|
| 8 |
+
"active_bytes_peak": 130559872512,
|
| 9 |
+
"allocated_bytes_current": 42499512320,
|
| 10 |
+
"allocated_bytes_peak": 127448441856,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 20966598656,
|
| 13 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 155661107200,
|
| 21 |
+
"reserved_bytes_peak": 155661107200,
|
| 22 |
+
"segment_count_current": 1104,
|
| 23 |
+
"segment_count_peak": 1104,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:06.418730Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42499512320,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127448441856,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42499512320,
|
| 33 |
+
"active_bytes_peak": 130559872512,
|
| 34 |
+
"allocated_bytes_current": 42499512320,
|
| 35 |
+
"allocated_bytes_peak": 127448441856,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 20966598656,
|
| 38 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 155661107200,
|
| 46 |
+
"reserved_bytes_peak": 155661107200,
|
| 47 |
+
"segment_count_current": 1104,
|
| 48 |
+
"segment_count_peak": 1104,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:05.830255Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20966598656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155661107200,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155661107200,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1104,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1104,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:06.429430Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:73415e2e740862df2997998353ec237668184603fe0563123d817019e4d518b7
|
| 3 |
+
size 402457
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 42499512320,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 130559872512,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 42499512320,
|
| 8 |
+
"active_bytes_peak": 130559872512,
|
| 9 |
+
"allocated_bytes_current": 42499512320,
|
| 10 |
+
"allocated_bytes_peak": 127448441856,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 20966598656,
|
| 13 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 155661107200,
|
| 21 |
+
"reserved_bytes_peak": 155661107200,
|
| 22 |
+
"segment_count_current": 1104,
|
| 23 |
+
"segment_count_peak": 1104,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:04:07.024118Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 42499512320,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 127448441856,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 42499512320,
|
| 33 |
+
"active_bytes_peak": 130559872512,
|
| 34 |
+
"allocated_bytes_current": 42499512320,
|
| 35 |
+
"allocated_bytes_peak": 127448441856,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 20966598656,
|
| 38 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 155661107200,
|
| 46 |
+
"reserved_bytes_peak": 155661107200,
|
| 47 |
+
"segment_count_current": 1104,
|
| 48 |
+
"segment_count_peak": 1104,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:04:06.430521Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 20966598656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 47197853696,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 155661107200,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 155661107200,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1104,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1104,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:04:07.033515Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a0671291f458477c03377803b0d7950ec7597b71f5fde4da7621bb7edcce385
|
| 3 |
+
size 402457
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=DSV2_LITE_L24
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=better-memory-dump-support
|
| 6 |
+
commit=b7c52442
|
| 7 |
+
num_layers=24
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=10944
|
| 10 |
+
moe_ffn_hidden=1408
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=6
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=8650752
|
| 15 |
+
heads=16
|
| 16 |
+
kv_groups=0
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=2816
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=1
|
| 22 |
+
dense_first_layers=1
|
| 23 |
+
moe_layers=23
|
| 24 |
+
rotary_base=10000
|
| 25 |
+
rotary_scaling=40
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=seq_aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=49152
|
| 39 |
+
mean_tokens_per_expert=6144
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.25
|
| 44 |
+
cuda_graph=1
|
| 45 |
+
probe_mode=0
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=memory
|
| 48 |
+
memory_dump_mode=iteration
|
| 49 |
+
memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=6
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|