granularity results echo/tierbmem @ 2026-08-26T19:10:38Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 3)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.json +79 -0
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9d24c9da0aac98760181ec8f6f17923ffdf043960281198b845b188095a05a0a
|
| 3 |
+
size 1706466
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72085230592,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 99464647168,
|
| 5 |
+
"active_bytes_peak_delta": 4390016000,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72085230592,
|
| 8 |
+
"active_bytes_peak": 99464647168,
|
| 9 |
+
"allocated_bytes_current": 72085230592,
|
| 10 |
+
"allocated_bytes_peak": 95074631168,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1931652096,
|
| 13 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 108192071680,
|
| 21 |
+
"reserved_bytes_peak": 108192071680,
|
| 22 |
+
"segment_count_current": 1166,
|
| 23 |
+
"segment_count_peak": 1166,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.149539Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72085230592,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 95074631168,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72068187136,
|
| 33 |
+
"active_bytes_peak": 95074631168,
|
| 34 |
+
"allocated_bytes_current": 72068187136,
|
| 35 |
+
"allocated_bytes_peak": 95074631168,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 2028387328,
|
| 38 |
+
"inactive_split_bytes_peak": 4951808512,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 100717821952,
|
| 46 |
+
"reserved_bytes_peak": 100717821952,
|
| 47 |
+
"segment_count_current": 1144,
|
| 48 |
+
"segment_count_peak": 1144,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.597941Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1931652096,
|
| 55 |
+
"inactive_split_bytes_current_delta": -96735232,
|
| 56 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 717435392,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108192071680,
|
| 68 |
+
"reserved_bytes_current_delta": 7474249728,
|
| 69 |
+
"reserved_bytes_peak": 108192071680,
|
| 70 |
+
"reserved_bytes_peak_delta": 7474249728,
|
| 71 |
+
"segment_count_current": 1166,
|
| 72 |
+
"segment_count_current_delta": 22,
|
| 73 |
+
"segment_count_peak": 1166,
|
| 74 |
+
"segment_count_peak_delta": 22,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.185975Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c8aed3b8a6cbd2f1450357cf223e62221bd6bbf5dc29d0ed5316ccec631d2a37
|
| 3 |
+
size 2102583
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72085230592,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99464647168,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72085230592,
|
| 8 |
+
"active_bytes_peak": 99464647168,
|
| 9 |
+
"allocated_bytes_current": 72085230592,
|
| 10 |
+
"allocated_bytes_peak": 95074631168,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1931652096,
|
| 13 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 108192071680,
|
| 21 |
+
"reserved_bytes_peak": 108192071680,
|
| 22 |
+
"segment_count_current": 1166,
|
| 23 |
+
"segment_count_peak": 1166,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.377676Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72085230592,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95074631168,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72085230592,
|
| 33 |
+
"active_bytes_peak": 99464647168,
|
| 34 |
+
"allocated_bytes_current": 72085230592,
|
| 35 |
+
"allocated_bytes_peak": 95074631168,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1931652096,
|
| 38 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 108192071680,
|
| 46 |
+
"reserved_bytes_peak": 108192071680,
|
| 47 |
+
"segment_count_current": 1166,
|
| 48 |
+
"segment_count_peak": 1166,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.190648Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1931652096,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108192071680,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108192071680,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1166,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1166,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.385355Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b05a572663b534cc1983bb0494ec9d597f0839b445405b6b9594fe4cdde71f30
|
| 3 |
+
size 331653
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72085230592,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99464647168,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72085230592,
|
| 8 |
+
"active_bytes_peak": 99464647168,
|
| 9 |
+
"allocated_bytes_current": 72085230592,
|
| 10 |
+
"allocated_bytes_peak": 95074631168,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 1931652096,
|
| 13 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 108192071680,
|
| 21 |
+
"reserved_bytes_peak": 108192071680,
|
| 22 |
+
"segment_count_current": 1166,
|
| 23 |
+
"segment_count_peak": 1166,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.567795Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72085230592,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95074631168,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72085230592,
|
| 33 |
+
"active_bytes_peak": 99464647168,
|
| 34 |
+
"allocated_bytes_current": 72085230592,
|
| 35 |
+
"allocated_bytes_peak": 95074631168,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 1931652096,
|
| 38 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 108192071680,
|
| 46 |
+
"reserved_bytes_peak": 108192071680,
|
| 47 |
+
"segment_count_current": 1166,
|
| 48 |
+
"segment_count_peak": 1166,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.386363Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1931652096,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5669243904,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108192071680,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108192071680,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1166,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1166,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.575238Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:caf72eb6a4500079fb7a27beb13c68170b12803ce2b1ee37ffca8f8dc3d46af3
|
| 3 |
+
size 331653
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72067058688,
|
| 3 |
+
"active_bytes_current_delta": 31381090304,
|
| 4 |
+
"active_bytes_peak": 74050460672,
|
| 5 |
+
"active_bytes_peak_delta": 33326743552,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72067058688,
|
| 8 |
+
"active_bytes_peak": 74050460672,
|
| 9 |
+
"allocated_bytes_current": 72067058688,
|
| 10 |
+
"allocated_bytes_peak": 74050460672,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1679291392,
|
| 13 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 84494254080,
|
| 21 |
+
"reserved_bytes_peak": 84494254080,
|
| 22 |
+
"segment_count_current": 1046,
|
| 23 |
+
"segment_count_peak": 1046,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:53.656629Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72067058688,
|
| 28 |
+
"allocated_bytes_current_delta": 31381090304,
|
| 29 |
+
"allocated_bytes_peak": 74050460672,
|
| 30 |
+
"allocated_bytes_peak_delta": 33326743552,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 40685968384,
|
| 33 |
+
"active_bytes_peak": 40723717120,
|
| 34 |
+
"allocated_bytes_current": 40685968384,
|
| 35 |
+
"allocated_bytes_peak": 40723717120,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 283993088,
|
| 38 |
+
"inactive_split_bytes_peak": 504194048,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 43130028032,
|
| 46 |
+
"reserved_bytes_peak": 43130028032,
|
| 47 |
+
"segment_count_current": 251,
|
| 48 |
+
"segment_count_peak": 251,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:14.578912Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1679291392,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1395298304,
|
| 56 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 4427889152,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 84494254080,
|
| 68 |
+
"reserved_bytes_current_delta": 41364226048,
|
| 69 |
+
"reserved_bytes_peak": 84494254080,
|
| 70 |
+
"reserved_bytes_peak_delta": 41364226048,
|
| 71 |
+
"segment_count_current": 1046,
|
| 72 |
+
"segment_count_current_delta": 795,
|
| 73 |
+
"segment_count_peak": 1046,
|
| 74 |
+
"segment_count_peak_delta": 795,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:53.692854Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9974cd72c5633b571d5ce6856e67df6dafcda06ff0059ce8a11a706bc3fa2877
|
| 3 |
+
size 1796624
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72067058688,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95073281024,
|
| 5 |
+
"active_bytes_peak_delta": 21022820352,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72067058688,
|
| 8 |
+
"active_bytes_peak": 95073281024,
|
| 9 |
+
"allocated_bytes_current": 72067058688,
|
| 10 |
+
"allocated_bytes_peak": 95073281024,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1974989824,
|
| 13 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 100428414976,
|
| 21 |
+
"reserved_bytes_peak": 100428414976,
|
| 22 |
+
"segment_count_current": 1145,
|
| 23 |
+
"segment_count_peak": 1145,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.287965Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72067058688,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95073281024,
|
| 30 |
+
"allocated_bytes_peak_delta": 21022820352,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72067058688,
|
| 33 |
+
"active_bytes_peak": 74050460672,
|
| 34 |
+
"allocated_bytes_current": 72067058688,
|
| 35 |
+
"allocated_bytes_peak": 74050460672,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1679291392,
|
| 38 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 84494254080,
|
| 46 |
+
"reserved_bytes_peak": 84494254080,
|
| 47 |
+
"segment_count_current": 1046,
|
| 48 |
+
"segment_count_peak": 1046,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:53.697327Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1974989824,
|
| 55 |
+
"inactive_split_bytes_current_delta": 295698432,
|
| 56 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 100428414976,
|
| 68 |
+
"reserved_bytes_current_delta": 15934160896,
|
| 69 |
+
"reserved_bytes_peak": 100428414976,
|
| 70 |
+
"reserved_bytes_peak_delta": 15934160896,
|
| 71 |
+
"segment_count_current": 1145,
|
| 72 |
+
"segment_count_current_delta": 99,
|
| 73 |
+
"segment_count_peak": 1145,
|
| 74 |
+
"segment_count_peak_delta": 99,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.320452Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6e1342c84c3066cf8a9e8a290ce04b4218ac538ebcc5633c1372fef50bc9d88f
|
| 3 |
+
size 1695924
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72067058688,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95073281024,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72067058688,
|
| 8 |
+
"active_bytes_peak": 95073281024,
|
| 9 |
+
"allocated_bytes_current": 72067058688,
|
| 10 |
+
"allocated_bytes_peak": 95073281024,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1974989824,
|
| 13 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 101252595712,
|
| 21 |
+
"reserved_bytes_peak": 101252595712,
|
| 22 |
+
"segment_count_current": 1146,
|
| 23 |
+
"segment_count_peak": 1146,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.565080Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72067058688,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95073281024,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72067058688,
|
| 33 |
+
"active_bytes_peak": 95073281024,
|
| 34 |
+
"allocated_bytes_current": 72067058688,
|
| 35 |
+
"allocated_bytes_peak": 95073281024,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1974989824,
|
| 38 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 100428414976,
|
| 46 |
+
"reserved_bytes_peak": 100428414976,
|
| 47 |
+
"segment_count_current": 1145,
|
| 48 |
+
"segment_count_peak": 1145,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.324314Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1974989824,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 101252595712,
|
| 68 |
+
"reserved_bytes_current_delta": 824180736,
|
| 69 |
+
"reserved_bytes_peak": 101252595712,
|
| 70 |
+
"reserved_bytes_peak_delta": 824180736,
|
| 71 |
+
"segment_count_current": 1146,
|
| 72 |
+
"segment_count_current_delta": 1,
|
| 73 |
+
"segment_count_peak": 1146,
|
| 74 |
+
"segment_count_peak_delta": 1,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.593215Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0fbb0d9fbd45546f2a956bbe610efb15010ac85ccea61be642a9307391993f15
|
| 3 |
+
size 1709822
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72084102144,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 99463518720,
|
| 5 |
+
"active_bytes_peak_delta": 4390237696,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72084102144,
|
| 8 |
+
"active_bytes_peak": 99463518720,
|
| 9 |
+
"allocated_bytes_current": 72084102144,
|
| 10 |
+
"allocated_bytes_peak": 95073281024,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1874060288,
|
| 13 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 108137545728,
|
| 21 |
+
"reserved_bytes_peak": 108137545728,
|
| 22 |
+
"segment_count_current": 1166,
|
| 23 |
+
"segment_count_peak": 1166,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.149582Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72084102144,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 95073281024,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72067058688,
|
| 33 |
+
"active_bytes_peak": 95073281024,
|
| 34 |
+
"allocated_bytes_current": 72067058688,
|
| 35 |
+
"allocated_bytes_peak": 95073281024,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1974989824,
|
| 38 |
+
"inactive_split_bytes_peak": 4932083200,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 101252595712,
|
| 46 |
+
"reserved_bytes_peak": 101252595712,
|
| 47 |
+
"segment_count_current": 1146,
|
| 48 |
+
"segment_count_peak": 1146,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.596691Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1874060288,
|
| 55 |
+
"inactive_split_bytes_current_delta": -100929536,
|
| 56 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 677471744,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108137545728,
|
| 68 |
+
"reserved_bytes_current_delta": 6884950016,
|
| 69 |
+
"reserved_bytes_peak": 108137545728,
|
| 70 |
+
"reserved_bytes_peak_delta": 6884950016,
|
| 71 |
+
"segment_count_current": 1166,
|
| 72 |
+
"segment_count_current_delta": 20,
|
| 73 |
+
"segment_count_peak": 1166,
|
| 74 |
+
"segment_count_peak_delta": 20,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.186358Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f07126cc4e9526702c515dfd2355fbcc4ce529a364dc88dd48932a58c562dcc8
|
| 3 |
+
size 1933754
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72084102144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99463518720,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72084102144,
|
| 8 |
+
"active_bytes_peak": 99463518720,
|
| 9 |
+
"allocated_bytes_current": 72084102144,
|
| 10 |
+
"allocated_bytes_peak": 95073281024,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1874060288,
|
| 13 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 108137545728,
|
| 21 |
+
"reserved_bytes_peak": 108137545728,
|
| 22 |
+
"segment_count_current": 1166,
|
| 23 |
+
"segment_count_peak": 1166,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.377683Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72084102144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95073281024,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72084102144,
|
| 33 |
+
"active_bytes_peak": 99463518720,
|
| 34 |
+
"allocated_bytes_current": 72084102144,
|
| 35 |
+
"allocated_bytes_peak": 95073281024,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1874060288,
|
| 38 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 108137545728,
|
| 46 |
+
"reserved_bytes_peak": 108137545728,
|
| 47 |
+
"segment_count_current": 1166,
|
| 48 |
+
"segment_count_peak": 1166,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.190467Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1874060288,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108137545728,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108137545728,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1166,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1166,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.385340Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d83f263d9ad0ad4dd8dcf3ad71c2ed2b4e16c45d8d739f8e02fdacd3110084b1
|
| 3 |
+
size 307966
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72084102144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99463518720,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72084102144,
|
| 8 |
+
"active_bytes_peak": 99463518720,
|
| 9 |
+
"allocated_bytes_current": 72084102144,
|
| 10 |
+
"allocated_bytes_peak": 95073281024,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 1874060288,
|
| 13 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 108137545728,
|
| 21 |
+
"reserved_bytes_peak": 108137545728,
|
| 22 |
+
"segment_count_current": 1166,
|
| 23 |
+
"segment_count_peak": 1166,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.567869Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72084102144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95073281024,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72084102144,
|
| 33 |
+
"active_bytes_peak": 99463518720,
|
| 34 |
+
"allocated_bytes_current": 72084102144,
|
| 35 |
+
"allocated_bytes_peak": 95073281024,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1874060288,
|
| 38 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 108137545728,
|
| 46 |
+
"reserved_bytes_peak": 108137545728,
|
| 47 |
+
"segment_count_current": 1166,
|
| 48 |
+
"segment_count_peak": 1166,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.386351Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1874060288,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5609554944,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108137545728,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108137545728,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1166,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1166,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.575890Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1c9bd3551ce330ca7aa01ba1c3f9ba3a2793be0da75f2393ed0483f90232bcaf
|
| 3 |
+
size 307935
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 31390646784,
|
| 4 |
+
"active_bytes_peak": 74041179136,
|
| 5 |
+
"active_bytes_peak_delta": 33317462016,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 74041179136,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 74041179136,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1835409920,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 85135982592,
|
| 21 |
+
"reserved_bytes_peak": 85135982592,
|
| 22 |
+
"segment_count_current": 1021,
|
| 23 |
+
"segment_count_peak": 1021,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:53.656598Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 31390646784,
|
| 29 |
+
"allocated_bytes_peak": 74041179136,
|
| 30 |
+
"allocated_bytes_peak_delta": 33317462016,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 40685968384,
|
| 33 |
+
"active_bytes_peak": 40723717120,
|
| 34 |
+
"allocated_bytes_current": 40685968384,
|
| 35 |
+
"allocated_bytes_peak": 40723717120,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 508388352,
|
| 38 |
+
"inactive_split_bytes_peak": 728589312,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 43864031232,
|
| 46 |
+
"reserved_bytes_peak": 43864031232,
|
| 47 |
+
"segment_count_current": 261,
|
| 48 |
+
"segment_count_peak": 261,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:14.578843Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1835409920,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1327021568,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 4327574016,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 85135982592,
|
| 68 |
+
"reserved_bytes_current_delta": 41271951360,
|
| 69 |
+
"reserved_bytes_peak": 85135982592,
|
| 70 |
+
"reserved_bytes_peak_delta": 41271951360,
|
| 71 |
+
"segment_count_current": 1021,
|
| 72 |
+
"segment_count_current_delta": 760,
|
| 73 |
+
"segment_count_peak": 1021,
|
| 74 |
+
"segment_count_peak_delta": 760,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:53.693208Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b3a60d377640686a765e132ac35e04aa64458b689a830bafe90b8657ffdb7cf1
|
| 3 |
+
size 2050482
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95064244224,
|
| 5 |
+
"active_bytes_peak_delta": 21023065088,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 95064244224,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 95064244224,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 2131108352,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 100415832064,
|
| 21 |
+
"reserved_bytes_peak": 100415832064,
|
| 22 |
+
"segment_count_current": 1115,
|
| 23 |
+
"segment_count_peak": 1115,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.287978Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95064244224,
|
| 30 |
+
"allocated_bytes_peak_delta": 21023065088,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 74041179136,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 74041179136,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1835409920,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 85135982592,
|
| 46 |
+
"reserved_bytes_peak": 85135982592,
|
| 47 |
+
"segment_count_current": 1021,
|
| 48 |
+
"segment_count_peak": 1021,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:53.697502Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2131108352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 295698432,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 100415832064,
|
| 68 |
+
"reserved_bytes_current_delta": 15279849472,
|
| 69 |
+
"reserved_bytes_peak": 100415832064,
|
| 70 |
+
"reserved_bytes_peak_delta": 15279849472,
|
| 71 |
+
"segment_count_current": 1115,
|
| 72 |
+
"segment_count_current_delta": 94,
|
| 73 |
+
"segment_count_peak": 1115,
|
| 74 |
+
"segment_count_peak_delta": 94,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.320860Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3a19a0a93d892ee5ad86e5c3730c7d4bde91f9d647376d6d844828a6589a47f3
|
| 3 |
+
size 1669210
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95064244224,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 95064244224,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 95064244224,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 2131108352,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 100415832064,
|
| 21 |
+
"reserved_bytes_peak": 100415832064,
|
| 22 |
+
"segment_count_current": 1115,
|
| 23 |
+
"segment_count_peak": 1115,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.565074Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95064244224,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 95064244224,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 95064244224,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 2131108352,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 100415832064,
|
| 46 |
+
"reserved_bytes_peak": 100415832064,
|
| 47 |
+
"segment_count_current": 1115,
|
| 48 |
+
"segment_count_peak": 1115,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.324647Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2131108352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 100415832064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 100415832064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1115,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1115,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.593327Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fe92993356aa1c104dce5ece68955e5b920b360d36efcc461ffb5424a5ef8b9b
|
| 3 |
+
size 1659945
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72093658624,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 99473075200,
|
| 5 |
+
"active_bytes_peak_delta": 4408830976,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72093658624,
|
| 8 |
+
"active_bytes_peak": 99473075200,
|
| 9 |
+
"allocated_bytes_current": 72093658624,
|
| 10 |
+
"allocated_bytes_peak": 95069056000,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 2030178816,
|
| 13 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 108303220736,
|
| 21 |
+
"reserved_bytes_peak": 108303220736,
|
| 22 |
+
"segment_count_current": 1158,
|
| 23 |
+
"segment_count_peak": 1158,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.149547Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72093658624,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 95069056000,
|
| 30 |
+
"allocated_bytes_peak_delta": 4811776,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 95064244224,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 95064244224,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 2131108352,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 100415832064,
|
| 46 |
+
"reserved_bytes_peak": 100415832064,
|
| 47 |
+
"segment_count_current": 1115,
|
| 48 |
+
"segment_count_peak": 1115,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.596724Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2030178816,
|
| 55 |
+
"inactive_split_bytes_current_delta": -100929536,
|
| 56 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 747258880,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108303220736,
|
| 68 |
+
"reserved_bytes_current_delta": 7887388672,
|
| 69 |
+
"reserved_bytes_peak": 108303220736,
|
| 70 |
+
"reserved_bytes_peak_delta": 7887388672,
|
| 71 |
+
"segment_count_current": 1158,
|
| 72 |
+
"segment_count_current_delta": 43,
|
| 73 |
+
"segment_count_peak": 1158,
|
| 74 |
+
"segment_count_peak_delta": 43,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.184118Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:13f830338122f0b64b45e22a5f1604c02d4f492f2b439322fa1273892a31c950
|
| 3 |
+
size 1929358
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72093658624,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99473075200,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72093658624,
|
| 8 |
+
"active_bytes_peak": 99473075200,
|
| 9 |
+
"allocated_bytes_current": 72093658624,
|
| 10 |
+
"allocated_bytes_peak": 95069056000,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 2030178816,
|
| 13 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 108303220736,
|
| 21 |
+
"reserved_bytes_peak": 108303220736,
|
| 22 |
+
"segment_count_current": 1158,
|
| 23 |
+
"segment_count_peak": 1158,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.377701Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72093658624,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95069056000,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72093658624,
|
| 33 |
+
"active_bytes_peak": 99473075200,
|
| 34 |
+
"allocated_bytes_current": 72093658624,
|
| 35 |
+
"allocated_bytes_peak": 95069056000,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 2030178816,
|
| 38 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 108303220736,
|
| 46 |
+
"reserved_bytes_peak": 108303220736,
|
| 47 |
+
"segment_count_current": 1158,
|
| 48 |
+
"segment_count_peak": 1158,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.188145Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2030178816,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108303220736,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108303220736,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1158,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1158,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.385243Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:30bb8b8c390915527946035eb877a81d8e81b0ca011a02f951384fd09a6d3623
|
| 3 |
+
size 304763
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72093658624,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99473075200,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72093658624,
|
| 8 |
+
"active_bytes_peak": 99473075200,
|
| 9 |
+
"allocated_bytes_current": 72093658624,
|
| 10 |
+
"allocated_bytes_peak": 95069056000,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 2030178816,
|
| 13 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 108303220736,
|
| 21 |
+
"reserved_bytes_peak": 108303220736,
|
| 22 |
+
"segment_count_current": 1158,
|
| 23 |
+
"segment_count_peak": 1158,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.567824Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72093658624,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95069056000,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72093658624,
|
| 33 |
+
"active_bytes_peak": 99473075200,
|
| 34 |
+
"allocated_bytes_current": 72093658624,
|
| 35 |
+
"allocated_bytes_peak": 95069056000,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 2030178816,
|
| 38 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 108303220736,
|
| 46 |
+
"reserved_bytes_peak": 108303220736,
|
| 47 |
+
"segment_count_current": 1158,
|
| 48 |
+
"segment_count_peak": 1158,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.386233Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2030178816,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108303220736,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108303220736,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1158,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1158,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.575037Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:442e464633ddb74b52d832884d0d5808bfd5f065f0c596eb24804e6d095d9442
|
| 3 |
+
size 304732
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 31390646784,
|
| 4 |
+
"active_bytes_peak": 74041179136,
|
| 5 |
+
"active_bytes_peak_delta": 33317462016,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 74041179136,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 74041179136,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1835409920,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 84255178752,
|
| 21 |
+
"reserved_bytes_peak": 84255178752,
|
| 22 |
+
"segment_count_current": 1018,
|
| 23 |
+
"segment_count_peak": 1018,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:53.656779Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 31390646784,
|
| 29 |
+
"allocated_bytes_peak": 74041179136,
|
| 30 |
+
"allocated_bytes_peak_delta": 33317462016,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 40685968384,
|
| 33 |
+
"active_bytes_peak": 40723717120,
|
| 34 |
+
"allocated_bytes_current": 40685968384,
|
| 35 |
+
"allocated_bytes_peak": 40723717120,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 508388352,
|
| 38 |
+
"inactive_split_bytes_peak": 728589312,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 43864031232,
|
| 46 |
+
"reserved_bytes_peak": 43864031232,
|
| 47 |
+
"segment_count_current": 261,
|
| 48 |
+
"segment_count_peak": 261,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:14.578764Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1835409920,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1327021568,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 4327574016,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 84255178752,
|
| 68 |
+
"reserved_bytes_current_delta": 40391147520,
|
| 69 |
+
"reserved_bytes_peak": 84255178752,
|
| 70 |
+
"reserved_bytes_peak_delta": 40391147520,
|
| 71 |
+
"segment_count_current": 1018,
|
| 72 |
+
"segment_count_current_delta": 757,
|
| 73 |
+
"segment_count_peak": 1018,
|
| 74 |
+
"segment_count_peak_delta": 757,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:53.694479Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:40dc7eed517b5ebfbe4f9051bfe06ee8e4753f811601822a4a2226b0a87057d2
|
| 3 |
+
size 2049975
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95064244224,
|
| 5 |
+
"active_bytes_peak_delta": 21023065088,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 95064244224,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 95064244224,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 2131108352,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 100709433344,
|
| 21 |
+
"reserved_bytes_peak": 100709433344,
|
| 22 |
+
"segment_count_current": 1116,
|
| 23 |
+
"segment_count_peak": 1116,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.287961Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95064244224,
|
| 30 |
+
"allocated_bytes_peak_delta": 21023065088,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 74041179136,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 74041179136,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1835409920,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 84255178752,
|
| 46 |
+
"reserved_bytes_peak": 84255178752,
|
| 47 |
+
"segment_count_current": 1018,
|
| 48 |
+
"segment_count_peak": 1018,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:53.698779Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2131108352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 295698432,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 100709433344,
|
| 68 |
+
"reserved_bytes_current_delta": 16454254592,
|
| 69 |
+
"reserved_bytes_peak": 100709433344,
|
| 70 |
+
"reserved_bytes_peak_delta": 16454254592,
|
| 71 |
+
"segment_count_current": 1116,
|
| 72 |
+
"segment_count_current_delta": 98,
|
| 73 |
+
"segment_count_peak": 1116,
|
| 74 |
+
"segment_count_peak_delta": 98,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.320885Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:85e3e8ed2bfb9ab468796aae78baa71b03b8237dab69340a405d06da59f3e7ca
|
| 3 |
+
size 1673137
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95064244224,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 95064244224,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 95064244224,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 2131108352,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 100709433344,
|
| 21 |
+
"reserved_bytes_peak": 100709433344,
|
| 22 |
+
"segment_count_current": 1116,
|
| 23 |
+
"segment_count_peak": 1116,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.565074Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95064244224,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 95064244224,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 95064244224,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 2131108352,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 100709433344,
|
| 46 |
+
"reserved_bytes_peak": 100709433344,
|
| 47 |
+
"segment_count_current": 1116,
|
| 48 |
+
"segment_count_peak": 1116,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.324586Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2131108352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 100709433344,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 100709433344,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1116,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1116,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.594182Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5fce0c4117776b38d996deabb0758da5b8b39d2c492b81ae68bc4504910185c4
|
| 3 |
+
size 1669731
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72093658624,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 99473075200,
|
| 5 |
+
"active_bytes_peak_delta": 4408830976,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72093658624,
|
| 8 |
+
"active_bytes_peak": 99473075200,
|
| 9 |
+
"allocated_bytes_current": 72093658624,
|
| 10 |
+
"allocated_bytes_peak": 95069056000,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 2030178816,
|
| 13 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 108303220736,
|
| 21 |
+
"reserved_bytes_peak": 108303220736,
|
| 22 |
+
"segment_count_current": 1158,
|
| 23 |
+
"segment_count_peak": 1158,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.149563Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72093658624,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 95069056000,
|
| 30 |
+
"allocated_bytes_peak_delta": 4811776,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 95064244224,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 95064244224,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 2131108352,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 100709433344,
|
| 46 |
+
"reserved_bytes_peak": 100709433344,
|
| 47 |
+
"segment_count_current": 1116,
|
| 48 |
+
"segment_count_peak": 1116,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.597543Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2030178816,
|
| 55 |
+
"inactive_split_bytes_current_delta": -100929536,
|
| 56 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 803881984,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108303220736,
|
| 68 |
+
"reserved_bytes_current_delta": 7593787392,
|
| 69 |
+
"reserved_bytes_peak": 108303220736,
|
| 70 |
+
"reserved_bytes_peak_delta": 7593787392,
|
| 71 |
+
"segment_count_current": 1158,
|
| 72 |
+
"segment_count_current_delta": 42,
|
| 73 |
+
"segment_count_peak": 1158,
|
| 74 |
+
"segment_count_peak_delta": 42,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.185386Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:036e2442bcd51d9d76f99f3882c13d6cab46d83cc9c568862a3e08e8c9aa24f8
|
| 3 |
+
size 1929444
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72093658624,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99473075200,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72093658624,
|
| 8 |
+
"active_bytes_peak": 99473075200,
|
| 9 |
+
"allocated_bytes_current": 72093658624,
|
| 10 |
+
"allocated_bytes_peak": 95069056000,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 2030178816,
|
| 13 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 108303220736,
|
| 21 |
+
"reserved_bytes_peak": 108303220736,
|
| 22 |
+
"segment_count_current": 1158,
|
| 23 |
+
"segment_count_peak": 1158,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.377662Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72093658624,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95069056000,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72093658624,
|
| 33 |
+
"active_bytes_peak": 99473075200,
|
| 34 |
+
"allocated_bytes_current": 72093658624,
|
| 35 |
+
"allocated_bytes_peak": 95069056000,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 2030178816,
|
| 38 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 108303220736,
|
| 46 |
+
"reserved_bytes_peak": 108303220736,
|
| 47 |
+
"segment_count_current": 1158,
|
| 48 |
+
"segment_count_peak": 1158,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.189138Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2030178816,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108303220736,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108303220736,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1158,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1158,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.384905Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2992fad4156e1d350878212bb7a63982fbb2b56096c61356234d0445f67d7bad
|
| 3 |
+
size 304872
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72093658624,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 99473075200,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72093658624,
|
| 8 |
+
"active_bytes_peak": 99473075200,
|
| 9 |
+
"allocated_bytes_current": 72093658624,
|
| 10 |
+
"allocated_bytes_peak": 95069056000,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 2030178816,
|
| 13 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 108303220736,
|
| 21 |
+
"reserved_bytes_peak": 108303220736,
|
| 22 |
+
"segment_count_current": 1158,
|
| 23 |
+
"segment_count_peak": 1158,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.567820Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72093658624,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95069056000,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72093658624,
|
| 33 |
+
"active_bytes_peak": 99473075200,
|
| 34 |
+
"allocated_bytes_current": 72093658624,
|
| 35 |
+
"allocated_bytes_peak": 95069056000,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 2030178816,
|
| 38 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 108303220736,
|
| 46 |
+
"reserved_bytes_peak": 108303220736,
|
| 47 |
+
"segment_count_current": 1158,
|
| 48 |
+
"segment_count_peak": 1158,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:56.385924Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2030178816,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5860045312,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108303220736,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 108303220736,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1158,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1158,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.575099Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:39fae1522f7b4a64dd46e1d572997c4344de54a5d9f75590fcb817f089c395ee
|
| 3 |
+
size 304841
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 31390646784,
|
| 4 |
+
"active_bytes_peak": 74041179136,
|
| 5 |
+
"active_bytes_peak_delta": 33317462016,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 74041179136,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 74041179136,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1835409920,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 84255178752,
|
| 21 |
+
"reserved_bytes_peak": 84255178752,
|
| 22 |
+
"segment_count_current": 1018,
|
| 23 |
+
"segment_count_peak": 1018,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:53.656602Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 31390646784,
|
| 29 |
+
"allocated_bytes_peak": 74041179136,
|
| 30 |
+
"allocated_bytes_peak_delta": 33317462016,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 40685968384,
|
| 33 |
+
"active_bytes_peak": 40723717120,
|
| 34 |
+
"allocated_bytes_current": 40685968384,
|
| 35 |
+
"allocated_bytes_peak": 40723717120,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 508388352,
|
| 38 |
+
"inactive_split_bytes_peak": 728589312,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 43864031232,
|
| 46 |
+
"reserved_bytes_peak": 43864031232,
|
| 47 |
+
"segment_count_current": 261,
|
| 48 |
+
"segment_count_peak": 261,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:14.578992Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1835409920,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1327021568,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 4327574016,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 84255178752,
|
| 68 |
+
"reserved_bytes_current_delta": 40391147520,
|
| 69 |
+
"reserved_bytes_peak": 84255178752,
|
| 70 |
+
"reserved_bytes_peak_delta": 40391147520,
|
| 71 |
+
"segment_count_current": 1018,
|
| 72 |
+
"segment_count_current_delta": 757,
|
| 73 |
+
"segment_count_peak": 1018,
|
| 74 |
+
"segment_count_peak_delta": 757,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:53.693328Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c67267c2569542f43a21e5a0d76ce414c2aea750e7ab0709b11e2b441f3193fd
|
| 3 |
+
size 2049975
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95064244224,
|
| 5 |
+
"active_bytes_peak_delta": 21023065088,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 95064244224,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 95064244224,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 2131108352,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 99828629504,
|
| 21 |
+
"reserved_bytes_peak": 99828629504,
|
| 22 |
+
"segment_count_current": 1113,
|
| 23 |
+
"segment_count_peak": 1113,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.287945Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95064244224,
|
| 30 |
+
"allocated_bytes_peak_delta": 21023065088,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 74041179136,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 74041179136,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1835409920,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 84255178752,
|
| 46 |
+
"reserved_bytes_peak": 84255178752,
|
| 47 |
+
"segment_count_current": 1018,
|
| 48 |
+
"segment_count_peak": 1018,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:53.697869Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2131108352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 295698432,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 99828629504,
|
| 68 |
+
"reserved_bytes_current_delta": 15573450752,
|
| 69 |
+
"reserved_bytes_peak": 99828629504,
|
| 70 |
+
"reserved_bytes_peak_delta": 15573450752,
|
| 71 |
+
"segment_count_current": 1113,
|
| 72 |
+
"segment_count_current_delta": 95,
|
| 73 |
+
"segment_count_peak": 1113,
|
| 74 |
+
"segment_count_peak_delta": 95,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.320175Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:910bc477e0ff1b4ba1c368ef808734a91bbdb16149c456dc613ad8a0de656026
|
| 3 |
+
size 1669041
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72076615168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 95064244224,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72076615168,
|
| 8 |
+
"active_bytes_peak": 95064244224,
|
| 9 |
+
"allocated_bytes_current": 72076615168,
|
| 10 |
+
"allocated_bytes_peak": 95064244224,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 2131108352,
|
| 13 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 99828629504,
|
| 21 |
+
"reserved_bytes_peak": 99828629504,
|
| 22 |
+
"segment_count_current": 1113,
|
| 23 |
+
"segment_count_peak": 1113,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:54.565581Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72076615168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 95064244224,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 95064244224,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 95064244224,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 2131108352,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 99828629504,
|
| 46 |
+
"reserved_bytes_peak": 99828629504,
|
| 47 |
+
"segment_count_current": 1113,
|
| 48 |
+
"segment_count_peak": 1113,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.324156Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2131108352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 99828629504,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 99828629504,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1113,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1113,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:54.600995Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3be16039d71d6c2058771a91dcac57af5c4f4c1798269998227160f87b9b348c
|
| 3 |
+
size 1659589
|
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 72093658624,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 99473075200,
|
| 5 |
+
"active_bytes_peak_delta": 4408830976,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 72093658624,
|
| 8 |
+
"active_bytes_peak": 99473075200,
|
| 9 |
+
"allocated_bytes_current": 72093658624,
|
| 10 |
+
"allocated_bytes_peak": 95069056000,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 2030178816,
|
| 13 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 108303220736,
|
| 21 |
+
"reserved_bytes_peak": 108303220736,
|
| 22 |
+
"segment_count_current": 1158,
|
| 23 |
+
"segment_count_peak": 1158,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:07:56.149558Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 72093658624,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 95069056000,
|
| 30 |
+
"allocated_bytes_peak_delta": 4811776,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 72076615168,
|
| 33 |
+
"active_bytes_peak": 95064244224,
|
| 34 |
+
"allocated_bytes_current": 72076615168,
|
| 35 |
+
"allocated_bytes_peak": 95064244224,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 2131108352,
|
| 38 |
+
"inactive_split_bytes_peak": 5056163328,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 99828629504,
|
| 46 |
+
"reserved_bytes_peak": 99828629504,
|
| 47 |
+
"segment_count_current": 1113,
|
| 48 |
+
"segment_count_peak": 1113,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:07:54.605114Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2030178816,
|
| 55 |
+
"inactive_split_bytes_current_delta": -100929536,
|
| 56 |
+
"inactive_split_bytes_peak": 5803422208,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 747258880,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 108303220736,
|
| 68 |
+
"reserved_bytes_current_delta": 8474591232,
|
| 69 |
+
"reserved_bytes_peak": 108303220736,
|
| 70 |
+
"reserved_bytes_peak_delta": 8474591232,
|
| 71 |
+
"segment_count_current": 1158,
|
| 72 |
+
"segment_count_current_delta": 45,
|
| 73 |
+
"segment_count_peak": 1158,
|
| 74 |
+
"segment_count_peak_delta": 45,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:07:56.186479Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|