granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 3)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.json +79 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e1ecd78195dd2b51e2dcd4598c8defa74ed9b7ae05ef2be9f31a36339e085ba
|
| 3 |
+
size 9942271
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84649044992,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 129025703424,
|
| 5 |
+
"active_bytes_peak_delta": 6617244160,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84649044992,
|
| 8 |
+
"active_bytes_peak": 129025703424,
|
| 9 |
+
"allocated_bytes_current": 84649044992,
|
| 10 |
+
"allocated_bytes_peak": 122408459264,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4209382400,
|
| 13 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 142709096448,
|
| 21 |
+
"reserved_bytes_peak": 142709096448,
|
| 22 |
+
"segment_count_current": 3356,
|
| 23 |
+
"segment_count_peak": 3356,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.321469Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84649044992,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 122408459264,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84632001536,
|
| 33 |
+
"active_bytes_peak": 122408459264,
|
| 34 |
+
"allocated_bytes_current": 84632001536,
|
| 35 |
+
"allocated_bytes_peak": 122408459264,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4173997056,
|
| 38 |
+
"inactive_split_bytes_peak": 12200414208,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 123394326528,
|
| 46 |
+
"reserved_bytes_peak": 123394326528,
|
| 47 |
+
"segment_count_current": 2923,
|
| 48 |
+
"segment_count_peak": 2923,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:17.862485Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4209382400,
|
| 55 |
+
"inactive_split_bytes_current_delta": 35385344,
|
| 56 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1072425984,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142709096448,
|
| 68 |
+
"reserved_bytes_current_delta": 19314769920,
|
| 69 |
+
"reserved_bytes_peak": 142709096448,
|
| 70 |
+
"reserved_bytes_peak_delta": 19314769920,
|
| 71 |
+
"segment_count_current": 3356,
|
| 72 |
+
"segment_count_current_delta": 433,
|
| 73 |
+
"segment_count_peak": 3356,
|
| 74 |
+
"segment_count_peak_delta": 433,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.536837Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dd3ad7429d7eeaad8c56aa12c5c1898f9108ed6966ff5f03b7d59c26d6891993
|
| 3 |
+
size 13044981
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84649044992,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129025703424,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84649044992,
|
| 8 |
+
"active_bytes_peak": 129025703424,
|
| 9 |
+
"allocated_bytes_current": 84649044992,
|
| 10 |
+
"allocated_bytes_peak": 122408459264,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4209382400,
|
| 13 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 142709096448,
|
| 21 |
+
"reserved_bytes_peak": 142709096448,
|
| 22 |
+
"segment_count_current": 3356,
|
| 23 |
+
"segment_count_peak": 3356,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.966678Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84649044992,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122408459264,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84649044992,
|
| 33 |
+
"active_bytes_peak": 129025703424,
|
| 34 |
+
"allocated_bytes_current": 84649044992,
|
| 35 |
+
"allocated_bytes_peak": 122408459264,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4209382400,
|
| 38 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 142709096448,
|
| 46 |
+
"reserved_bytes_peak": 142709096448,
|
| 47 |
+
"segment_count_current": 3356,
|
| 48 |
+
"segment_count_peak": 3356,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.565839Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4209382400,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142709096448,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142709096448,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3356,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3356,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.991948Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:11bace237037948288f21ac0d282aa1371521ae5a556b3a40f8884d69666e01d
|
| 3 |
+
size 1147818
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84649044992,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129025703424,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84649044992,
|
| 8 |
+
"active_bytes_peak": 129025703424,
|
| 9 |
+
"allocated_bytes_current": 84649044992,
|
| 10 |
+
"allocated_bytes_peak": 122408459264,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4209382400,
|
| 13 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 142709096448,
|
| 21 |
+
"reserved_bytes_peak": 142709096448,
|
| 22 |
+
"segment_count_current": 3356,
|
| 23 |
+
"segment_count_peak": 3356,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:22.392483Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84649044992,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122408459264,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84649044992,
|
| 33 |
+
"active_bytes_peak": 129025703424,
|
| 34 |
+
"allocated_bytes_current": 84649044992,
|
| 35 |
+
"allocated_bytes_peak": 122408459264,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4209382400,
|
| 38 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 142709096448,
|
| 46 |
+
"reserved_bytes_peak": 142709096448,
|
| 47 |
+
"segment_count_current": 3356,
|
| 48 |
+
"segment_count_peak": 3356,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.993666Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4209382400,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13272840192,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142709096448,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142709096448,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3356,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3356,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:22.416360Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:80d60f38ec3abfba66f7d751f65e7bb4c0db57b61be034f1c8dda382a731d4b0
|
| 3 |
+
size 1147818
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84623316992,
|
| 3 |
+
"active_bytes_current_delta": 41069773824,
|
| 4 |
+
"active_bytes_peak": 91617038336,
|
| 5 |
+
"active_bytes_peak_delta": 48057203712,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84623316992,
|
| 8 |
+
"active_bytes_peak": 91617038336,
|
| 9 |
+
"allocated_bytes_current": 84623316992,
|
| 10 |
+
"allocated_bytes_peak": 91617038336,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4312705024,
|
| 13 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 93296001024,
|
| 21 |
+
"reserved_bytes_peak": 93296001024,
|
| 22 |
+
"segment_count_current": 2187,
|
| 23 |
+
"segment_count_peak": 2187,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:14.715878Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84623316992,
|
| 28 |
+
"allocated_bytes_current_delta": 41069773824,
|
| 29 |
+
"allocated_bytes_peak": 91617038336,
|
| 30 |
+
"allocated_bytes_peak_delta": 48057203712,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43553543168,
|
| 33 |
+
"active_bytes_peak": 43559834624,
|
| 34 |
+
"allocated_bytes_current": 43553543168,
|
| 35 |
+
"allocated_bytes_peak": 43559834624,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 1256303616,
|
| 38 |
+
"inactive_split_bytes_peak": 1612912640,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 44835012608,
|
| 46 |
+
"reserved_bytes_peak": 44835012608,
|
| 47 |
+
"segment_count_current": 883,
|
| 48 |
+
"segment_count_peak": 883,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:56:36.124259Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4312705024,
|
| 55 |
+
"inactive_split_bytes_current_delta": 3056401408,
|
| 56 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10569281024,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 93296001024,
|
| 68 |
+
"reserved_bytes_current_delta": 48460988416,
|
| 69 |
+
"reserved_bytes_peak": 93296001024,
|
| 70 |
+
"reserved_bytes_peak_delta": 48460988416,
|
| 71 |
+
"segment_count_current": 2187,
|
| 72 |
+
"segment_count_current_delta": 1304,
|
| 73 |
+
"segment_count_peak": 2187,
|
| 74 |
+
"segment_count_peak_delta": 1304,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:14.902087Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b37c5ff79eea33f8e00c37e16a0c30121fae013bda203537694c76d3dd26ef1d
|
| 3 |
+
size 9776530
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84623316992,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122401423360,
|
| 5 |
+
"active_bytes_peak_delta": 30784385024,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84623316992,
|
| 8 |
+
"active_bytes_peak": 122401423360,
|
| 9 |
+
"allocated_bytes_current": 84623316992,
|
| 10 |
+
"allocated_bytes_peak": 122401423360,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4214138880,
|
| 13 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 124092678144,
|
| 21 |
+
"reserved_bytes_peak": 124092678144,
|
| 22 |
+
"segment_count_current": 2896,
|
| 23 |
+
"segment_count_peak": 2896,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:16.268085Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84623316992,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122401423360,
|
| 30 |
+
"allocated_bytes_peak_delta": 30784385024,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84623316992,
|
| 33 |
+
"active_bytes_peak": 91617038336,
|
| 34 |
+
"allocated_bytes_current": 84623316992,
|
| 35 |
+
"allocated_bytes_peak": 91617038336,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 4312705024,
|
| 38 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 93296001024,
|
| 46 |
+
"reserved_bytes_peak": 93296001024,
|
| 47 |
+
"segment_count_current": 2187,
|
| 48 |
+
"segment_count_peak": 2187,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:14.936634Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4214138880,
|
| 55 |
+
"inactive_split_bytes_current_delta": -98566144,
|
| 56 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124092678144,
|
| 68 |
+
"reserved_bytes_current_delta": 30796677120,
|
| 69 |
+
"reserved_bytes_peak": 124092678144,
|
| 70 |
+
"reserved_bytes_peak_delta": 30796677120,
|
| 71 |
+
"segment_count_current": 2896,
|
| 72 |
+
"segment_count_current_delta": 709,
|
| 73 |
+
"segment_count_peak": 2896,
|
| 74 |
+
"segment_count_peak_delta": 709,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:16.452869Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0ed47c8b35e9481a7efdee5e8b9467741a891e79a9557961f971e65adc8030b
|
| 3 |
+
size 9503132
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84622792704,
|
| 3 |
+
"active_bytes_current_delta": -524288,
|
| 4 |
+
"active_bytes_peak": 122862320640,
|
| 5 |
+
"active_bytes_peak_delta": 460897280,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84622792704,
|
| 8 |
+
"active_bytes_peak": 122862320640,
|
| 9 |
+
"allocated_bytes_current": 84622792704,
|
| 10 |
+
"allocated_bytes_peak": 122409335808,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4418086912,
|
| 13 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 124092678144,
|
| 21 |
+
"reserved_bytes_peak": 124092678144,
|
| 22 |
+
"segment_count_current": 2896,
|
| 23 |
+
"segment_count_peak": 2896,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:17.636808Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84622792704,
|
| 28 |
+
"allocated_bytes_current_delta": -524288,
|
| 29 |
+
"allocated_bytes_peak": 122409335808,
|
| 30 |
+
"allocated_bytes_peak_delta": 7912448,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84623316992,
|
| 33 |
+
"active_bytes_peak": 122401423360,
|
| 34 |
+
"allocated_bytes_current": 84623316992,
|
| 35 |
+
"allocated_bytes_peak": 122401423360,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 4214138880,
|
| 38 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 124092678144,
|
| 46 |
+
"reserved_bytes_peak": 124092678144,
|
| 47 |
+
"segment_count_current": 2896,
|
| 48 |
+
"segment_count_peak": 2896,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:16.477699Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4418086912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 203948032,
|
| 56 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124092678144,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 124092678144,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2896,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2896,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:17.824121Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dc955e4105c6e2315399dc8ef2b4a81962c2f9b2d7105e2abfd4462d12b09629
|
| 3 |
+
size 9479267
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84640360448,
|
| 3 |
+
"active_bytes_current_delta": 17567744,
|
| 4 |
+
"active_bytes_peak": 129017018880,
|
| 5 |
+
"active_bytes_peak_delta": 6154698240,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84640360448,
|
| 8 |
+
"active_bytes_peak": 129017018880,
|
| 9 |
+
"allocated_bytes_current": 84640360448,
|
| 10 |
+
"allocated_bytes_peak": 122409335808,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4350187520,
|
| 13 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 142943977472,
|
| 21 |
+
"reserved_bytes_peak": 142943977472,
|
| 22 |
+
"segment_count_current": 3336,
|
| 23 |
+
"segment_count_peak": 3336,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.321537Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84640360448,
|
| 28 |
+
"allocated_bytes_current_delta": 17567744,
|
| 29 |
+
"allocated_bytes_peak": 122409335808,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84622792704,
|
| 33 |
+
"active_bytes_peak": 122862320640,
|
| 34 |
+
"allocated_bytes_current": 84622792704,
|
| 35 |
+
"allocated_bytes_peak": 122409335808,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 4418086912,
|
| 38 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 124092678144,
|
| 46 |
+
"reserved_bytes_peak": 124092678144,
|
| 47 |
+
"segment_count_current": 2896,
|
| 48 |
+
"segment_count_peak": 2896,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:17.848657Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4350187520,
|
| 55 |
+
"inactive_split_bytes_current_delta": -67899392,
|
| 56 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1132885504,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142943977472,
|
| 68 |
+
"reserved_bytes_current_delta": 18851299328,
|
| 69 |
+
"reserved_bytes_peak": 142943977472,
|
| 70 |
+
"reserved_bytes_peak_delta": 18851299328,
|
| 71 |
+
"segment_count_current": 3336,
|
| 72 |
+
"segment_count_current_delta": 440,
|
| 73 |
+
"segment_count_peak": 3336,
|
| 74 |
+
"segment_count_peak_delta": 440,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.523444Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2e81a30934d574b230fd9fcb7a0b0a469de4855c57b57d22ecb26f7ff3030996
|
| 3 |
+
size 11659206
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84640360448,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129017018880,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84640360448,
|
| 8 |
+
"active_bytes_peak": 129017018880,
|
| 9 |
+
"allocated_bytes_current": 84640360448,
|
| 10 |
+
"allocated_bytes_peak": 122409335808,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4350187520,
|
| 13 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 142943977472,
|
| 21 |
+
"reserved_bytes_peak": 142943977472,
|
| 22 |
+
"segment_count_current": 3336,
|
| 23 |
+
"segment_count_peak": 3336,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.966674Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84640360448,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122409335808,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84640360448,
|
| 33 |
+
"active_bytes_peak": 129017018880,
|
| 34 |
+
"allocated_bytes_current": 84640360448,
|
| 35 |
+
"allocated_bytes_peak": 122409335808,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 4350187520,
|
| 38 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 142943977472,
|
| 46 |
+
"reserved_bytes_peak": 142943977472,
|
| 47 |
+
"segment_count_current": 3336,
|
| 48 |
+
"segment_count_peak": 3336,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.550555Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4350187520,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142943977472,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142943977472,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3336,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3336,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.990538Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d065d6e7a7a58cce740e1eb2653e0a7fb3332951ac483721044a3a3280b0af5c
|
| 3 |
+
size 1024383
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84640360448,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129017018880,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84640360448,
|
| 8 |
+
"active_bytes_peak": 129017018880,
|
| 9 |
+
"allocated_bytes_current": 84640360448,
|
| 10 |
+
"allocated_bytes_peak": 122409335808,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4350187520,
|
| 13 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 142943977472,
|
| 21 |
+
"reserved_bytes_peak": 142943977472,
|
| 22 |
+
"segment_count_current": 3336,
|
| 23 |
+
"segment_count_peak": 3336,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:22.392470Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84640360448,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122409335808,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84640360448,
|
| 33 |
+
"active_bytes_peak": 129017018880,
|
| 34 |
+
"allocated_bytes_current": 84640360448,
|
| 35 |
+
"allocated_bytes_peak": 122409335808,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 4350187520,
|
| 38 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 142943977472,
|
| 46 |
+
"reserved_bytes_peak": 142943977472,
|
| 47 |
+
"segment_count_current": 3336,
|
| 48 |
+
"segment_count_peak": 3336,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.992237Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4350187520,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142943977472,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142943977472,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3336,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3336,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:22.413897Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0d40a12d5ca3f1a42a5aeb79a48e78cc0e6b77a16c8a8a15c856f8f778fc1552
|
| 3 |
+
size 1024414
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84639218688,
|
| 3 |
+
"active_bytes_current_delta": 41082396160,
|
| 4 |
+
"active_bytes_peak": 91614024192,
|
| 5 |
+
"active_bytes_peak_delta": 48050910208,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84639218688,
|
| 8 |
+
"active_bytes_peak": 91614024192,
|
| 9 |
+
"allocated_bytes_current": 84639218688,
|
| 10 |
+
"allocated_bytes_peak": 91614024192,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 4003202048,
|
| 13 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 93308583936,
|
| 21 |
+
"reserved_bytes_peak": 93308583936,
|
| 22 |
+
"segment_count_current": 2209,
|
| 23 |
+
"segment_count_peak": 2209,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:14.715878Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84639218688,
|
| 28 |
+
"allocated_bytes_current_delta": 41082396160,
|
| 29 |
+
"allocated_bytes_peak": 91614024192,
|
| 30 |
+
"allocated_bytes_peak_delta": 48050910208,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43556822528,
|
| 33 |
+
"active_bytes_peak": 43563113984,
|
| 34 |
+
"allocated_bytes_current": 43556822528,
|
| 35 |
+
"allocated_bytes_peak": 43563113984,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1223664128,
|
| 38 |
+
"inactive_split_bytes_peak": 1612912640,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 44795166720,
|
| 46 |
+
"reserved_bytes_peak": 44795166720,
|
| 47 |
+
"segment_count_current": 887,
|
| 48 |
+
"segment_count_peak": 887,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:56:36.124364Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4003202048,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2779537920,
|
| 56 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10580878336,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 93308583936,
|
| 68 |
+
"reserved_bytes_current_delta": 48513417216,
|
| 69 |
+
"reserved_bytes_peak": 93308583936,
|
| 70 |
+
"reserved_bytes_peak_delta": 48513417216,
|
| 71 |
+
"segment_count_current": 2209,
|
| 72 |
+
"segment_count_current_delta": 1322,
|
| 73 |
+
"segment_count_peak": 2209,
|
| 74 |
+
"segment_count_peak_delta": 1322,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:14.911068Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8bbc327b800ea28c61c5b117e64d564f9029b195268fced181206659829598e5
|
| 3 |
+
size 11336704
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84639218688,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122380765184,
|
| 5 |
+
"active_bytes_peak_delta": 30766740992,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84639218688,
|
| 8 |
+
"active_bytes_peak": 122380765184,
|
| 9 |
+
"allocated_bytes_current": 84639218688,
|
| 10 |
+
"allocated_bytes_peak": 122380765184,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 3904635904,
|
| 13 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 124080095232,
|
| 21 |
+
"reserved_bytes_peak": 124080095232,
|
| 22 |
+
"segment_count_current": 2932,
|
| 23 |
+
"segment_count_peak": 2932,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:16.268107Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84639218688,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122380765184,
|
| 30 |
+
"allocated_bytes_peak_delta": 30766740992,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84639218688,
|
| 33 |
+
"active_bytes_peak": 91614024192,
|
| 34 |
+
"allocated_bytes_current": 84639218688,
|
| 35 |
+
"allocated_bytes_peak": 91614024192,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 4003202048,
|
| 38 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 93308583936,
|
| 46 |
+
"reserved_bytes_peak": 93308583936,
|
| 47 |
+
"segment_count_current": 2209,
|
| 48 |
+
"segment_count_peak": 2209,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:14.949036Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 3904635904,
|
| 55 |
+
"inactive_split_bytes_current_delta": -98566144,
|
| 56 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124080095232,
|
| 68 |
+
"reserved_bytes_current_delta": 30771511296,
|
| 69 |
+
"reserved_bytes_peak": 124080095232,
|
| 70 |
+
"reserved_bytes_peak_delta": 30771511296,
|
| 71 |
+
"segment_count_current": 2932,
|
| 72 |
+
"segment_count_current_delta": 723,
|
| 73 |
+
"segment_count_peak": 2932,
|
| 74 |
+
"segment_count_peak_delta": 723,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:16.465288Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f24e01ae2a41576be2f51d98cdcca72d4e4107f29c84218a35eeeace42684f1e
|
| 3 |
+
size 10005096
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84639218688,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122385467392,
|
| 5 |
+
"active_bytes_peak_delta": 4702208,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84639218688,
|
| 8 |
+
"active_bytes_peak": 122385467392,
|
| 9 |
+
"allocated_bytes_current": 84639218688,
|
| 10 |
+
"allocated_bytes_peak": 122385467392,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 4007396352,
|
| 13 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 124080095232,
|
| 21 |
+
"reserved_bytes_peak": 124080095232,
|
| 22 |
+
"segment_count_current": 2932,
|
| 23 |
+
"segment_count_peak": 2932,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:17.636816Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84639218688,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122385467392,
|
| 30 |
+
"allocated_bytes_peak_delta": 4702208,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84639218688,
|
| 33 |
+
"active_bytes_peak": 122380765184,
|
| 34 |
+
"allocated_bytes_current": 84639218688,
|
| 35 |
+
"allocated_bytes_peak": 122380765184,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 3904635904,
|
| 38 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 124080095232,
|
| 46 |
+
"reserved_bytes_peak": 124080095232,
|
| 47 |
+
"segment_count_current": 2932,
|
| 48 |
+
"segment_count_peak": 2932,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:16.493482Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4007396352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 102760448,
|
| 56 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124080095232,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 124080095232,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2932,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2932,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:17.835543Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bf995b39791b738cb9918361bf3dfeabbdb7755842c029e8c32e0679685a6dd1
|
| 3 |
+
size 9983320
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84656262144,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 129032920576,
|
| 5 |
+
"active_bytes_peak_delta": 6647453184,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84656262144,
|
| 8 |
+
"active_bytes_peak": 129032920576,
|
| 9 |
+
"allocated_bytes_current": 84656262144,
|
| 10 |
+
"allocated_bytes_peak": 122385467392,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 4042781696,
|
| 13 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 142549712896,
|
| 21 |
+
"reserved_bytes_peak": 142549712896,
|
| 22 |
+
"segment_count_current": 3353,
|
| 23 |
+
"segment_count_peak": 3353,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.321480Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84656262144,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 122385467392,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84639218688,
|
| 33 |
+
"active_bytes_peak": 122385467392,
|
| 34 |
+
"allocated_bytes_current": 84639218688,
|
| 35 |
+
"allocated_bytes_peak": 122385467392,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 4007396352,
|
| 38 |
+
"inactive_split_bytes_peak": 12193790976,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 124080095232,
|
| 46 |
+
"reserved_bytes_peak": 124080095232,
|
| 47 |
+
"segment_count_current": 2932,
|
| 48 |
+
"segment_count_peak": 2932,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:17.862945Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4042781696,
|
| 55 |
+
"inactive_split_bytes_current_delta": 35385344,
|
| 56 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 910351360,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142549712896,
|
| 68 |
+
"reserved_bytes_current_delta": 18469617664,
|
| 69 |
+
"reserved_bytes_peak": 142549712896,
|
| 70 |
+
"reserved_bytes_peak_delta": 18469617664,
|
| 71 |
+
"segment_count_current": 3353,
|
| 72 |
+
"segment_count_current_delta": 421,
|
| 73 |
+
"segment_count_peak": 3353,
|
| 74 |
+
"segment_count_peak_delta": 421,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.533677Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:40569b1347f21a5e50f7a875558837dc21417d48ee9975658ac7638eb7a0952f
|
| 3 |
+
size 13043697
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84656262144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129032920576,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84656262144,
|
| 8 |
+
"active_bytes_peak": 129032920576,
|
| 9 |
+
"allocated_bytes_current": 84656262144,
|
| 10 |
+
"allocated_bytes_peak": 122385467392,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 4042781696,
|
| 13 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 142549712896,
|
| 21 |
+
"reserved_bytes_peak": 142549712896,
|
| 22 |
+
"segment_count_current": 3353,
|
| 23 |
+
"segment_count_peak": 3353,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.966676Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84656262144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122385467392,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84656262144,
|
| 33 |
+
"active_bytes_peak": 129032920576,
|
| 34 |
+
"allocated_bytes_current": 84656262144,
|
| 35 |
+
"allocated_bytes_peak": 122385467392,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 4042781696,
|
| 38 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 142549712896,
|
| 46 |
+
"reserved_bytes_peak": 142549712896,
|
| 47 |
+
"segment_count_current": 3353,
|
| 48 |
+
"segment_count_peak": 3353,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.562190Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4042781696,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142549712896,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142549712896,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3353,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3353,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.991954Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b05b8fd9cfb442a68ac108921b2a52dc9cb45f3ace93957b3d1a37262674f1d4
|
| 3 |
+
size 1145785
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84656262144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129032920576,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84656262144,
|
| 8 |
+
"active_bytes_peak": 129032920576,
|
| 9 |
+
"allocated_bytes_current": 84656262144,
|
| 10 |
+
"allocated_bytes_peak": 122385467392,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 4042781696,
|
| 13 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 142549712896,
|
| 21 |
+
"reserved_bytes_peak": 142549712896,
|
| 22 |
+
"segment_count_current": 3353,
|
| 23 |
+
"segment_count_peak": 3353,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:22.392480Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84656262144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122385467392,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84656262144,
|
| 33 |
+
"active_bytes_peak": 129032920576,
|
| 34 |
+
"allocated_bytes_current": 84656262144,
|
| 35 |
+
"allocated_bytes_peak": 122385467392,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 4042781696,
|
| 38 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 142549712896,
|
| 46 |
+
"reserved_bytes_peak": 142549712896,
|
| 47 |
+
"segment_count_current": 3353,
|
| 48 |
+
"segment_count_peak": 3353,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.993694Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4042781696,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13104142336,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142549712896,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142549712896,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3353,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3353,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:22.415105Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7f80a025f020d7c7fa17f8345eb5bb620908d07eea41fae6730f7901da0fd6d3
|
| 3 |
+
size 1145785
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84620231168,
|
| 3 |
+
"active_bytes_current_delta": 41069553664,
|
| 4 |
+
"active_bytes_peak": 91608373760,
|
| 5 |
+
"active_bytes_peak_delta": 48051404800,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84620231168,
|
| 8 |
+
"active_bytes_peak": 91608373760,
|
| 9 |
+
"allocated_bytes_current": 84620231168,
|
| 10 |
+
"allocated_bytes_peak": 91608373760,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 4173184512,
|
| 13 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 93308583936,
|
| 21 |
+
"reserved_bytes_peak": 93308583936,
|
| 22 |
+
"segment_count_current": 2199,
|
| 23 |
+
"segment_count_peak": 2199,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:14.715883Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84620231168,
|
| 28 |
+
"allocated_bytes_current_delta": 41069553664,
|
| 29 |
+
"allocated_bytes_peak": 91608373760,
|
| 30 |
+
"allocated_bytes_peak_delta": 48051404800,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43550677504,
|
| 33 |
+
"active_bytes_peak": 43556968960,
|
| 34 |
+
"allocated_bytes_current": 43550677504,
|
| 35 |
+
"allocated_bytes_peak": 43556968960,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1252877824,
|
| 38 |
+
"inactive_split_bytes_peak": 1612912640,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 44820332544,
|
| 46 |
+
"reserved_bytes_peak": 44820332544,
|
| 47 |
+
"segment_count_current": 889,
|
| 48 |
+
"segment_count_peak": 889,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:56:36.124297Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4173184512,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2920306688,
|
| 56 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10776623104,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 93308583936,
|
| 68 |
+
"reserved_bytes_current_delta": 48488251392,
|
| 69 |
+
"reserved_bytes_peak": 93308583936,
|
| 70 |
+
"reserved_bytes_peak_delta": 48488251392,
|
| 71 |
+
"segment_count_current": 2199,
|
| 72 |
+
"segment_count_current_delta": 1310,
|
| 73 |
+
"segment_count_peak": 2199,
|
| 74 |
+
"segment_count_peak_delta": 1310,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:14.907724Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:808df123683f0e4d1437cfc18860e38f6b8f0e2a1b8f889185f461ff7d43a04a
|
| 3 |
+
size 10265593
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84620231168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122397165568,
|
| 5 |
+
"active_bytes_peak_delta": 30788791808,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84620231168,
|
| 8 |
+
"active_bytes_peak": 122397165568,
|
| 9 |
+
"allocated_bytes_current": 84620231168,
|
| 10 |
+
"allocated_bytes_peak": 122397165568,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 4074618368,
|
| 13 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 124111552512,
|
| 21 |
+
"reserved_bytes_peak": 124111552512,
|
| 22 |
+
"segment_count_current": 2919,
|
| 23 |
+
"segment_count_peak": 2919,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:16.268101Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84620231168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122397165568,
|
| 30 |
+
"allocated_bytes_peak_delta": 30788791808,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84620231168,
|
| 33 |
+
"active_bytes_peak": 91608373760,
|
| 34 |
+
"allocated_bytes_current": 84620231168,
|
| 35 |
+
"allocated_bytes_peak": 91608373760,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 4173184512,
|
| 38 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 93308583936,
|
| 46 |
+
"reserved_bytes_peak": 93308583936,
|
| 47 |
+
"segment_count_current": 2199,
|
| 48 |
+
"segment_count_peak": 2199,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:14.946209Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4074618368,
|
| 55 |
+
"inactive_split_bytes_current_delta": -98566144,
|
| 56 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124111552512,
|
| 68 |
+
"reserved_bytes_current_delta": 30802968576,
|
| 69 |
+
"reserved_bytes_peak": 124111552512,
|
| 70 |
+
"reserved_bytes_peak_delta": 30802968576,
|
| 71 |
+
"segment_count_current": 2919,
|
| 72 |
+
"segment_count_current_delta": 720,
|
| 73 |
+
"segment_count_peak": 2919,
|
| 74 |
+
"segment_count_peak_delta": 720,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:16.464169Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9546a3cdedc19d1379af2b642d67f7f7f9fe8f9d25155d6e6c41a6354f1d167e
|
| 3 |
+
size 9964097
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84620231168,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122566591488,
|
| 5 |
+
"active_bytes_peak_delta": 169425920,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84620231168,
|
| 8 |
+
"active_bytes_peak": 122566591488,
|
| 9 |
+
"allocated_bytes_current": 84620231168,
|
| 10 |
+
"allocated_bytes_peak": 122415596544,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 4177378816,
|
| 13 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 124111552512,
|
| 21 |
+
"reserved_bytes_peak": 124111552512,
|
| 22 |
+
"segment_count_current": 2919,
|
| 23 |
+
"segment_count_peak": 2919,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:17.636821Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84620231168,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122415596544,
|
| 30 |
+
"allocated_bytes_peak_delta": 18430976,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84620231168,
|
| 33 |
+
"active_bytes_peak": 122397165568,
|
| 34 |
+
"allocated_bytes_current": 84620231168,
|
| 35 |
+
"allocated_bytes_peak": 122397165568,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 4074618368,
|
| 38 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 124111552512,
|
| 46 |
+
"reserved_bytes_peak": 124111552512,
|
| 47 |
+
"segment_count_current": 2919,
|
| 48 |
+
"segment_count_peak": 2919,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:16.492021Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4177378816,
|
| 55 |
+
"inactive_split_bytes_current_delta": 102760448,
|
| 56 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124111552512,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 124111552512,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2919,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2919,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:17.832101Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a80831a0dd49d6d99c7c3777f30280b56c64c598e12b9b44375e59e393488bdf
|
| 3 |
+
size 9942396
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84637274624,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 129013933056,
|
| 5 |
+
"active_bytes_peak_delta": 6447341568,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84637274624,
|
| 8 |
+
"active_bytes_peak": 129013933056,
|
| 9 |
+
"allocated_bytes_current": 84637274624,
|
| 10 |
+
"allocated_bytes_peak": 122415596544,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 4212764160,
|
| 13 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 142700707840,
|
| 21 |
+
"reserved_bytes_peak": 142700707840,
|
| 22 |
+
"segment_count_current": 3344,
|
| 23 |
+
"segment_count_peak": 3344,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.321529Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84637274624,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 122415596544,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84620231168,
|
| 33 |
+
"active_bytes_peak": 122566591488,
|
| 34 |
+
"allocated_bytes_current": 84620231168,
|
| 35 |
+
"allocated_bytes_peak": 122415596544,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 4177378816,
|
| 38 |
+
"inactive_split_bytes_peak": 12389535744,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 124111552512,
|
| 46 |
+
"reserved_bytes_peak": 124111552512,
|
| 47 |
+
"segment_count_current": 2919,
|
| 48 |
+
"segment_count_peak": 2919,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:17.859724Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4212764160,
|
| 55 |
+
"inactive_split_bytes_current_delta": 35385344,
|
| 56 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 886686208,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142700707840,
|
| 68 |
+
"reserved_bytes_current_delta": 18589155328,
|
| 69 |
+
"reserved_bytes_peak": 142700707840,
|
| 70 |
+
"reserved_bytes_peak_delta": 18589155328,
|
| 71 |
+
"segment_count_current": 3344,
|
| 72 |
+
"segment_count_current_delta": 425,
|
| 73 |
+
"segment_count_peak": 3344,
|
| 74 |
+
"segment_count_peak_delta": 425,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.531953Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bbccb966f0bc7c697ff4f83958c98600e6bf894d49398b6d8004b29286261149
|
| 3 |
+
size 13044392
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84637274624,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129013933056,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84637274624,
|
| 8 |
+
"active_bytes_peak": 129013933056,
|
| 9 |
+
"allocated_bytes_current": 84637274624,
|
| 10 |
+
"allocated_bytes_peak": 122415596544,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 4212764160,
|
| 13 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 142700707840,
|
| 21 |
+
"reserved_bytes_peak": 142700707840,
|
| 22 |
+
"segment_count_current": 3344,
|
| 23 |
+
"segment_count_peak": 3344,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.966680Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84637274624,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122415596544,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84637274624,
|
| 33 |
+
"active_bytes_peak": 129013933056,
|
| 34 |
+
"allocated_bytes_current": 84637274624,
|
| 35 |
+
"allocated_bytes_peak": 122415596544,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 4212764160,
|
| 38 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 142700707840,
|
| 46 |
+
"reserved_bytes_peak": 142700707840,
|
| 47 |
+
"segment_count_current": 3344,
|
| 48 |
+
"segment_count_peak": 3344,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.560891Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4212764160,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142700707840,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142700707840,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3344,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3344,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.992261Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:32ca825932a591126c68aa69b765727b43e77abb7ea5e931820e0d7e6a33c27e
|
| 3 |
+
size 1146117
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84637274624,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 129013933056,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84637274624,
|
| 8 |
+
"active_bytes_peak": 129013933056,
|
| 9 |
+
"allocated_bytes_current": 84637274624,
|
| 10 |
+
"allocated_bytes_peak": 122415596544,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 4212764160,
|
| 13 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 142700707840,
|
| 21 |
+
"reserved_bytes_peak": 142700707840,
|
| 22 |
+
"segment_count_current": 3344,
|
| 23 |
+
"segment_count_peak": 3344,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:22.392443Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84637274624,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122415596544,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84637274624,
|
| 33 |
+
"active_bytes_peak": 129013933056,
|
| 34 |
+
"allocated_bytes_current": 84637274624,
|
| 35 |
+
"allocated_bytes_peak": 122415596544,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 4212764160,
|
| 38 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 142700707840,
|
| 46 |
+
"reserved_bytes_peak": 142700707840,
|
| 47 |
+
"segment_count_current": 3344,
|
| 48 |
+
"segment_count_peak": 3344,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:21.994010Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4212764160,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 13276221952,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 142700707840,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 142700707840,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 3344,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 3344,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:22.415679Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4b4a812c8c9953fd12d61a9fa1a3c060b55cf99bd397c61e08fddd1e5740f5aa
|
| 3 |
+
size 1146117
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84623316992,
|
| 3 |
+
"active_bytes_current_delta": 41069773824,
|
| 4 |
+
"active_bytes_peak": 91617038336,
|
| 5 |
+
"active_bytes_peak_delta": 48057203712,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84623316992,
|
| 8 |
+
"active_bytes_peak": 91617038336,
|
| 9 |
+
"allocated_bytes_current": 84623316992,
|
| 10 |
+
"allocated_bytes_peak": 91617038336,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 4312705024,
|
| 13 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 93296001024,
|
| 21 |
+
"reserved_bytes_peak": 93296001024,
|
| 22 |
+
"segment_count_current": 2187,
|
| 23 |
+
"segment_count_peak": 2187,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:14.715850Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84623316992,
|
| 28 |
+
"allocated_bytes_current_delta": 41069773824,
|
| 29 |
+
"allocated_bytes_peak": 91617038336,
|
| 30 |
+
"allocated_bytes_peak_delta": 48057203712,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43553543168,
|
| 33 |
+
"active_bytes_peak": 43559834624,
|
| 34 |
+
"allocated_bytes_current": 43553543168,
|
| 35 |
+
"allocated_bytes_peak": 43559834624,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1256303616,
|
| 38 |
+
"inactive_split_bytes_peak": 1612912640,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 44835012608,
|
| 46 |
+
"reserved_bytes_peak": 44835012608,
|
| 47 |
+
"segment_count_current": 883,
|
| 48 |
+
"segment_count_peak": 883,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:56:36.124260Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4312705024,
|
| 55 |
+
"inactive_split_bytes_current_delta": 3056401408,
|
| 56 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10569281024,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 93296001024,
|
| 68 |
+
"reserved_bytes_current_delta": 48460988416,
|
| 69 |
+
"reserved_bytes_peak": 93296001024,
|
| 70 |
+
"reserved_bytes_peak_delta": 48460988416,
|
| 71 |
+
"segment_count_current": 2187,
|
| 72 |
+
"segment_count_current_delta": 1304,
|
| 73 |
+
"segment_count_peak": 2187,
|
| 74 |
+
"segment_count_peak_delta": 1304,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:14.907500Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aed1b5fc9e78f2e9ca4e6fc8395b8a81c19d4cf802bf1f36ef818da1c4d22d8d
|
| 3 |
+
size 9776530
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84623316992,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122552418304,
|
| 5 |
+
"active_bytes_peak_delta": 30935379968,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84623316992,
|
| 8 |
+
"active_bytes_peak": 122552418304,
|
| 9 |
+
"allocated_bytes_current": 84623316992,
|
| 10 |
+
"allocated_bytes_peak": 122401423360,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 4314802176,
|
| 13 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 124092678144,
|
| 21 |
+
"reserved_bytes_peak": 124092678144,
|
| 22 |
+
"segment_count_current": 2896,
|
| 23 |
+
"segment_count_peak": 2896,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:16.268085Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84623316992,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122401423360,
|
| 30 |
+
"allocated_bytes_peak_delta": 30784385024,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84623316992,
|
| 33 |
+
"active_bytes_peak": 91617038336,
|
| 34 |
+
"allocated_bytes_current": 84623316992,
|
| 35 |
+
"allocated_bytes_peak": 91617038336,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 4312705024,
|
| 38 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 93296001024,
|
| 46 |
+
"reserved_bytes_peak": 93296001024,
|
| 47 |
+
"segment_count_current": 2187,
|
| 48 |
+
"segment_count_peak": 2187,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:14.941036Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4314802176,
|
| 55 |
+
"inactive_split_bytes_current_delta": 2097152,
|
| 56 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124092678144,
|
| 68 |
+
"reserved_bytes_current_delta": 30796677120,
|
| 69 |
+
"reserved_bytes_peak": 124092678144,
|
| 70 |
+
"reserved_bytes_peak_delta": 30796677120,
|
| 71 |
+
"segment_count_current": 2896,
|
| 72 |
+
"segment_count_current_delta": 709,
|
| 73 |
+
"segment_count_peak": 2896,
|
| 74 |
+
"segment_count_peak_delta": 709,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:16.452271Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dbbeb24eacfb24bdbd734c5ff0663e0aacb114568394a159437e97c2de5818cb
|
| 3 |
+
size 9503153
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84623316992,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 122862844928,
|
| 5 |
+
"active_bytes_peak_delta": 310426624,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84623316992,
|
| 8 |
+
"active_bytes_peak": 122862844928,
|
| 9 |
+
"allocated_bytes_current": 84623316992,
|
| 10 |
+
"allocated_bytes_peak": 122409860096,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 4518225920,
|
| 13 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 124092678144,
|
| 21 |
+
"reserved_bytes_peak": 124092678144,
|
| 22 |
+
"segment_count_current": 2896,
|
| 23 |
+
"segment_count_peak": 2896,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:17.636782Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84623316992,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 122409860096,
|
| 30 |
+
"allocated_bytes_peak_delta": 8436736,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84623316992,
|
| 33 |
+
"active_bytes_peak": 122552418304,
|
| 34 |
+
"allocated_bytes_current": 84623316992,
|
| 35 |
+
"allocated_bytes_peak": 122401423360,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 4314802176,
|
| 38 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 124092678144,
|
| 46 |
+
"reserved_bytes_peak": 124092678144,
|
| 47 |
+
"segment_count_current": 2896,
|
| 48 |
+
"segment_count_peak": 2896,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:16.476772Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4518225920,
|
| 55 |
+
"inactive_split_bytes_current_delta": 203423744,
|
| 56 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 124092678144,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 124092678144,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2896,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2896,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:17.824669Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e53e1b1d1fd8cded3d575fbc8221f1e543225d075d6c45b4726535d8c3f70dad
|
| 3 |
+
size 9478730
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84640360448,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 129017018880,
|
| 5 |
+
"active_bytes_peak_delta": 6154173952,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84640360448,
|
| 8 |
+
"active_bytes_peak": 129017018880,
|
| 9 |
+
"allocated_bytes_current": 84640360448,
|
| 10 |
+
"allocated_bytes_peak": 122409860096,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 4350187520,
|
| 13 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 143044640768,
|
| 21 |
+
"reserved_bytes_peak": 143044640768,
|
| 22 |
+
"segment_count_current": 3337,
|
| 23 |
+
"segment_count_peak": 3337,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:57:21.321493Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84640360448,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 122409860096,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84623316992,
|
| 33 |
+
"active_bytes_peak": 122862844928,
|
| 34 |
+
"allocated_bytes_current": 84623316992,
|
| 35 |
+
"allocated_bytes_peak": 122409860096,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 4518225920,
|
| 38 |
+
"inactive_split_bytes_peak": 12182193664,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 124092678144,
|
| 46 |
+
"reserved_bytes_peak": 124092678144,
|
| 47 |
+
"segment_count_current": 2896,
|
| 48 |
+
"segment_count_peak": 2896,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:57:17.848873Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4350187520,
|
| 55 |
+
"inactive_split_bytes_current_delta": -168038400,
|
| 56 |
+
"inactive_split_bytes_peak": 13315079168,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1132885504,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 143044640768,
|
| 68 |
+
"reserved_bytes_current_delta": 18951962624,
|
| 69 |
+
"reserved_bytes_peak": 143044640768,
|
| 70 |
+
"reserved_bytes_peak_delta": 18951962624,
|
| 71 |
+
"segment_count_current": 3337,
|
| 72 |
+
"segment_count_current_delta": 441,
|
| 73 |
+
"segment_count_peak": 3337,
|
| 74 |
+
"segment_count_peak_delta": 441,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:57:21.526139Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|