granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env +61 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log +0 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85305508352,
|
| 3 |
+
"active_bytes_current_delta": 40784898560,
|
| 4 |
+
"active_bytes_peak": 92964658688,
|
| 5 |
+
"active_bytes_peak_delta": 48418883072,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85305508352,
|
| 8 |
+
"active_bytes_peak": 92964658688,
|
| 9 |
+
"allocated_bytes_current": 85305508352,
|
| 10 |
+
"allocated_bytes_peak": 92964658688,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1738882560,
|
| 13 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 103718846464,
|
| 21 |
+
"reserved_bytes_peak": 103718846464,
|
| 22 |
+
"segment_count_current": 1773,
|
| 23 |
+
"segment_count_peak": 1773,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:13.624966Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85305508352,
|
| 28 |
+
"allocated_bytes_current_delta": 40784898560,
|
| 29 |
+
"allocated_bytes_peak": 92964658688,
|
| 30 |
+
"allocated_bytes_peak_delta": 48418883072,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 270362624,
|
| 38 |
+
"inactive_split_bytes_peak": 874342400,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 47949283328,
|
| 46 |
+
"reserved_bytes_peak": 47949283328,
|
| 47 |
+
"segment_count_current": 615,
|
| 48 |
+
"segment_count_peak": 615,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:52:35.069417Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1738882560,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1468519936,
|
| 56 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10642682880,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103718846464,
|
| 68 |
+
"reserved_bytes_current_delta": 55769563136,
|
| 69 |
+
"reserved_bytes_peak": 103718846464,
|
| 70 |
+
"reserved_bytes_peak_delta": 55769563136,
|
| 71 |
+
"segment_count_current": 1773,
|
| 72 |
+
"segment_count_current_delta": 1158,
|
| 73 |
+
"segment_count_peak": 1773,
|
| 74 |
+
"segment_count_peak_delta": 1158,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:13.678100Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:694ea38ba10338075e96eeda159409303d08f01f3ce6b6542de81a88c56d0973
|
| 3 |
+
size 3143234
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85305508352,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123848197632,
|
| 5 |
+
"active_bytes_peak_delta": 30883538944,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85305508352,
|
| 8 |
+
"active_bytes_peak": 123848197632,
|
| 9 |
+
"allocated_bytes_current": 85305508352,
|
| 10 |
+
"allocated_bytes_peak": 123848197632,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 2143632896,
|
| 13 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 131409641472,
|
| 21 |
+
"reserved_bytes_peak": 131409641472,
|
| 22 |
+
"segment_count_current": 1969,
|
| 23 |
+
"segment_count_peak": 1969,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.223997Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85305508352,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123848197632,
|
| 30 |
+
"allocated_bytes_peak_delta": 30883538944,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85305508352,
|
| 33 |
+
"active_bytes_peak": 92964658688,
|
| 34 |
+
"allocated_bytes_current": 85305508352,
|
| 35 |
+
"allocated_bytes_peak": 92964658688,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1738882560,
|
| 38 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 103718846464,
|
| 46 |
+
"reserved_bytes_peak": 103718846464,
|
| 47 |
+
"segment_count_current": 1773,
|
| 48 |
+
"segment_count_peak": 1773,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:13.684294Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2143632896,
|
| 55 |
+
"inactive_split_bytes_current_delta": 404750336,
|
| 56 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131409641472,
|
| 68 |
+
"reserved_bytes_current_delta": 27690795008,
|
| 69 |
+
"reserved_bytes_peak": 131409641472,
|
| 70 |
+
"reserved_bytes_peak_delta": 27690795008,
|
| 71 |
+
"segment_count_current": 1969,
|
| 72 |
+
"segment_count_current_delta": 196,
|
| 73 |
+
"segment_count_peak": 1969,
|
| 74 |
+
"segment_count_peak_delta": 196,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:14.273495Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:618c5c4132390d0279c26422aede52859c7e602d326c07c4f03d0c5c8b884014
|
| 3 |
+
size 2702011
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85305508352,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 125663821824,
|
| 5 |
+
"active_bytes_peak_delta": 1815624192,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85305508352,
|
| 8 |
+
"active_bytes_peak": 125663821824,
|
| 9 |
+
"allocated_bytes_current": 85305508352,
|
| 10 |
+
"allocated_bytes_peak": 123851882496,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 2344959488,
|
| 13 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 131409641472,
|
| 21 |
+
"reserved_bytes_peak": 131409641472,
|
| 22 |
+
"segment_count_current": 1969,
|
| 23 |
+
"segment_count_peak": 1969,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.987363Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85305508352,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123851882496,
|
| 30 |
+
"allocated_bytes_peak_delta": 3684864,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85305508352,
|
| 33 |
+
"active_bytes_peak": 123848197632,
|
| 34 |
+
"allocated_bytes_current": 85305508352,
|
| 35 |
+
"allocated_bytes_peak": 123848197632,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 2143632896,
|
| 38 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 131409641472,
|
| 46 |
+
"reserved_bytes_peak": 131409641472,
|
| 47 |
+
"segment_count_current": 1969,
|
| 48 |
+
"segment_count_peak": 1969,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:14.279853Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2344959488,
|
| 55 |
+
"inactive_split_bytes_current_delta": 201326592,
|
| 56 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131409641472,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 131409641472,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1969,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1969,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:15.035956Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3612085b1d534624bdcdb31884a77147d8b8f9b10729fd61ccaa8514993da8f4
|
| 3 |
+
size 2662107
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85322551808,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 131086775296,
|
| 5 |
+
"active_bytes_peak_delta": 5422953472,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85322551808,
|
| 8 |
+
"active_bytes_peak": 131086775296,
|
| 9 |
+
"allocated_bytes_current": 85322551808,
|
| 10 |
+
"allocated_bytes_peak": 123851882496,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1747004928,
|
| 13 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 144793665536,
|
| 21 |
+
"reserved_bytes_peak": 144793665536,
|
| 22 |
+
"segment_count_current": 1829,
|
| 23 |
+
"segment_count_peak": 1969,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:17.851076Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85322551808,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 123851882496,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85305508352,
|
| 33 |
+
"active_bytes_peak": 125663821824,
|
| 34 |
+
"allocated_bytes_current": 85305508352,
|
| 35 |
+
"allocated_bytes_peak": 123851882496,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 2344959488,
|
| 38 |
+
"inactive_split_bytes_peak": 11517025280,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 131409641472,
|
| 46 |
+
"reserved_bytes_peak": 131409641472,
|
| 47 |
+
"segment_count_current": 1969,
|
| 48 |
+
"segment_count_peak": 1969,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:15.042065Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1747004928,
|
| 55 |
+
"inactive_split_bytes_current_delta": -597954560,
|
| 56 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1442927104,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144793665536,
|
| 68 |
+
"reserved_bytes_current_delta": 13384024064,
|
| 69 |
+
"reserved_bytes_peak": 144793665536,
|
| 70 |
+
"reserved_bytes_peak_delta": 13384024064,
|
| 71 |
+
"segment_count_current": 1829,
|
| 72 |
+
"segment_count_current_delta": -140,
|
| 73 |
+
"segment_count_peak": 1969,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:17.905515Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6cd0d594d5d48552cf0c14b59f08361da894323b9a6a8a39e8e098115e24a667
|
| 3 |
+
size 3557570
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85322551808,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131086775296,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85322551808,
|
| 8 |
+
"active_bytes_peak": 131086775296,
|
| 9 |
+
"allocated_bytes_current": 85322551808,
|
| 10 |
+
"allocated_bytes_peak": 123851882496,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1747004928,
|
| 13 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 144793665536,
|
| 21 |
+
"reserved_bytes_peak": 144793665536,
|
| 22 |
+
"segment_count_current": 1829,
|
| 23 |
+
"segment_count_peak": 1969,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.232062Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85322551808,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123851882496,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85322551808,
|
| 33 |
+
"active_bytes_peak": 131086775296,
|
| 34 |
+
"allocated_bytes_current": 85322551808,
|
| 35 |
+
"allocated_bytes_peak": 123851882496,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1747004928,
|
| 38 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 144793665536,
|
| 46 |
+
"reserved_bytes_peak": 144793665536,
|
| 47 |
+
"segment_count_current": 1829,
|
| 48 |
+
"segment_count_peak": 1969,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:17.912423Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1747004928,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144793665536,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144793665536,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1829,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1969,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.242281Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d5e8a8bb0df40c64891d9ae6bfa2582ffdbc6a8ae7563522847f1564a50bb1ea
|
| 3 |
+
size 500333
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85322551808,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131086775296,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85322551808,
|
| 8 |
+
"active_bytes_peak": 131086775296,
|
| 9 |
+
"allocated_bytes_current": 85322551808,
|
| 10 |
+
"allocated_bytes_peak": 123851882496,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1747004928,
|
| 13 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 144793665536,
|
| 21 |
+
"reserved_bytes_peak": 144793665536,
|
| 22 |
+
"segment_count_current": 1829,
|
| 23 |
+
"segment_count_peak": 1969,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.558867Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85322551808,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123851882496,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85322551808,
|
| 33 |
+
"active_bytes_peak": 131086775296,
|
| 34 |
+
"allocated_bytes_current": 85322551808,
|
| 35 |
+
"allocated_bytes_peak": 123851882496,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1747004928,
|
| 38 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 144793665536,
|
| 46 |
+
"reserved_bytes_peak": 144793665536,
|
| 47 |
+
"segment_count_current": 1829,
|
| 48 |
+
"segment_count_peak": 1969,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:18.243412Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1747004928,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12959952384,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144793665536,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144793665536,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1829,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1969,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.568683Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:24e13dc2e7201bffa77a91b8f62cd8b56d6a5fe9d5ddcc5ce109389363fdfdcd
|
| 3 |
+
size 500333
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 40775332352,
|
| 4 |
+
"active_bytes_peak": 92952749568,
|
| 5 |
+
"active_bytes_peak_delta": 48406973952,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 92952749568,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 92952749568,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1486304768,
|
| 13 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 103574142976,
|
| 21 |
+
"reserved_bytes_peak": 103574142976,
|
| 22 |
+
"segment_count_current": 1763,
|
| 23 |
+
"segment_count_peak": 1763,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:13.624980Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 40775332352,
|
| 29 |
+
"allocated_bytes_peak": 92952749568,
|
| 30 |
+
"allocated_bytes_peak_delta": 48406973952,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 421357568,
|
| 38 |
+
"inactive_split_bytes_peak": 1025337344,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:52:35.069371Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1486304768,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1064947200,
|
| 56 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10830632448,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103574142976,
|
| 68 |
+
"reserved_bytes_current_delta": 55322869760,
|
| 69 |
+
"reserved_bytes_peak": 103574142976,
|
| 70 |
+
"reserved_bytes_peak_delta": 55322869760,
|
| 71 |
+
"segment_count_current": 1763,
|
| 72 |
+
"segment_count_current_delta": 1142,
|
| 73 |
+
"segment_count_peak": 1763,
|
| 74 |
+
"segment_count_peak_delta": 1142,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:13.676649Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e646bda37ca523679dba662256b554f942923991e427b8372b97ef4530a348ce
|
| 3 |
+
size 3134744
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123820125184,
|
| 5 |
+
"active_bytes_peak_delta": 30867375616,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 123820125184,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 123820125184,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1888957952,
|
| 13 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 131522887680,
|
| 21 |
+
"reserved_bytes_peak": 131522887680,
|
| 22 |
+
"segment_count_current": 1946,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.223968Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123820125184,
|
| 30 |
+
"allocated_bytes_peak_delta": 30867375616,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 92952749568,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 92952749568,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1486304768,
|
| 38 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 103574142976,
|
| 46 |
+
"reserved_bytes_peak": 103574142976,
|
| 47 |
+
"segment_count_current": 1763,
|
| 48 |
+
"segment_count_peak": 1763,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:13.682850Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1888957952,
|
| 55 |
+
"inactive_split_bytes_current_delta": 402653184,
|
| 56 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131522887680,
|
| 68 |
+
"reserved_bytes_current_delta": 27948744704,
|
| 69 |
+
"reserved_bytes_peak": 131522887680,
|
| 70 |
+
"reserved_bytes_peak_delta": 27948744704,
|
| 71 |
+
"segment_count_current": 1946,
|
| 72 |
+
"segment_count_current_delta": 183,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 183,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:14.543073Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d8f153e6fb4a65e1f916bdba14421432711badae1a069b1a3a02f72c5409007c
|
| 3 |
+
size 2573434
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 125632310272,
|
| 5 |
+
"active_bytes_peak_delta": 1812185088,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 125632310272,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 123820370944,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 2090284544,
|
| 13 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 131522887680,
|
| 21 |
+
"reserved_bytes_peak": 131522887680,
|
| 22 |
+
"segment_count_current": 1946,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.987380Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123820370944,
|
| 30 |
+
"allocated_bytes_peak_delta": 245760,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 123820125184,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 123820125184,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1888957952,
|
| 38 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 131522887680,
|
| 46 |
+
"reserved_bytes_peak": 131522887680,
|
| 47 |
+
"segment_count_current": 1946,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:14.550970Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2090284544,
|
| 55 |
+
"inactive_split_bytes_current_delta": 201326592,
|
| 56 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131522887680,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 131522887680,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1946,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:15.033243Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:31636bdd9a3a2ab584a046752635e338b392894ff07510af06acc175c3fe1dbd
|
| 3 |
+
size 2584909
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 5444898816,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1492329984,
|
| 13 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 144529424384,
|
| 21 |
+
"reserved_bytes_peak": 144529424384,
|
| 22 |
+
"segment_count_current": 1800,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:17.851087Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 9080832,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 125632310272,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 123820370944,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 2090284544,
|
| 38 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 131522887680,
|
| 46 |
+
"reserved_bytes_peak": 131522887680,
|
| 47 |
+
"segment_count_current": 1946,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:15.038665Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1492329984,
|
| 55 |
+
"inactive_split_bytes_current_delta": -597954560,
|
| 56 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 866884096,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144529424384,
|
| 68 |
+
"reserved_bytes_current_delta": 13006536704,
|
| 69 |
+
"reserved_bytes_peak": 144529424384,
|
| 70 |
+
"reserved_bytes_peak_delta": 13006536704,
|
| 71 |
+
"segment_count_current": 1800,
|
| 72 |
+
"segment_count_current_delta": -146,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:17.904663Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:61489bf2e007af0e6fc131c0df3d99a67bef57dfe023553044ebeb9e86e24783
|
| 3 |
+
size 3223552
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1492329984,
|
| 13 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 144529424384,
|
| 21 |
+
"reserved_bytes_peak": 144529424384,
|
| 22 |
+
"segment_count_current": 1800,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.232117Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312985600,
|
| 33 |
+
"active_bytes_peak": 131077209088,
|
| 34 |
+
"allocated_bytes_current": 85312985600,
|
| 35 |
+
"allocated_bytes_peak": 123829451776,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1492329984,
|
| 38 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 144529424384,
|
| 46 |
+
"reserved_bytes_peak": 144529424384,
|
| 47 |
+
"segment_count_current": 1800,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:17.910644Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1492329984,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144529424384,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144529424384,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1800,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.242332Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:421c3422903e64916555d3dda93bee8254beb070c1a142410b1a4c3f4d41dff7
|
| 3 |
+
size 490746
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1492329984,
|
| 13 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 144529424384,
|
| 21 |
+
"reserved_bytes_peak": 144529424384,
|
| 22 |
+
"segment_count_current": 1800,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.558884Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312985600,
|
| 33 |
+
"active_bytes_peak": 131077209088,
|
| 34 |
+
"allocated_bytes_current": 85312985600,
|
| 35 |
+
"allocated_bytes_peak": 123829451776,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1492329984,
|
| 38 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 144529424384,
|
| 46 |
+
"reserved_bytes_peak": 144529424384,
|
| 47 |
+
"segment_count_current": 1800,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:18.243448Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1492329984,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144529424384,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144529424384,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1800,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.569017Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6839aa3dab10f50ef418ffab11d273249c24a192aca1ebce4e714816aac48090
|
| 3 |
+
size 490746
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 40775332352,
|
| 4 |
+
"active_bytes_peak": 92952749568,
|
| 5 |
+
"active_bytes_peak_delta": 48406973952,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 92952749568,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 92952749568,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1486304768,
|
| 13 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 103574142976,
|
| 21 |
+
"reserved_bytes_peak": 103574142976,
|
| 22 |
+
"segment_count_current": 1763,
|
| 23 |
+
"segment_count_peak": 1763,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:13.624970Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 40775332352,
|
| 29 |
+
"allocated_bytes_peak": 92952749568,
|
| 30 |
+
"allocated_bytes_peak_delta": 48406973952,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 421357568,
|
| 38 |
+
"inactive_split_bytes_peak": 1025337344,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:52:35.069260Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1486304768,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1064947200,
|
| 56 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10830632448,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103574142976,
|
| 68 |
+
"reserved_bytes_current_delta": 55322869760,
|
| 69 |
+
"reserved_bytes_peak": 103574142976,
|
| 70 |
+
"reserved_bytes_peak_delta": 55322869760,
|
| 71 |
+
"segment_count_current": 1763,
|
| 72 |
+
"segment_count_current_delta": 1142,
|
| 73 |
+
"segment_count_peak": 1763,
|
| 74 |
+
"segment_count_peak_delta": 1142,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:13.677322Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:63fa1f05e4028667920d1b5bc237b672f59b4e6e98ddee231347d7614e526bc1
|
| 3 |
+
size 3134747
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123820125184,
|
| 5 |
+
"active_bytes_peak_delta": 30867375616,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 123820125184,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 123820125184,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1888957952,
|
| 13 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 131522887680,
|
| 21 |
+
"reserved_bytes_peak": 131522887680,
|
| 22 |
+
"segment_count_current": 1946,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.223959Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123820125184,
|
| 30 |
+
"allocated_bytes_peak_delta": 30867375616,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 92952749568,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 92952749568,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1486304768,
|
| 38 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 103574142976,
|
| 46 |
+
"reserved_bytes_peak": 103574142976,
|
| 47 |
+
"segment_count_current": 1763,
|
| 48 |
+
"segment_count_peak": 1763,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:13.683157Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1888957952,
|
| 55 |
+
"inactive_split_bytes_current_delta": 402653184,
|
| 56 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131522887680,
|
| 68 |
+
"reserved_bytes_current_delta": 27948744704,
|
| 69 |
+
"reserved_bytes_peak": 131522887680,
|
| 70 |
+
"reserved_bytes_peak_delta": 27948744704,
|
| 71 |
+
"segment_count_current": 1946,
|
| 72 |
+
"segment_count_current_delta": 183,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 183,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:14.500901Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:031c2f06e89858824f1d85cc94f40af6b4435b68367a820ac81ea6dc14b0cef8
|
| 3 |
+
size 2577463
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 125632310272,
|
| 5 |
+
"active_bytes_peak_delta": 1812185088,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 125632310272,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 123820370944,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 2090284544,
|
| 13 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 131522887680,
|
| 21 |
+
"reserved_bytes_peak": 131522887680,
|
| 22 |
+
"segment_count_current": 1946,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.987309Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123820370944,
|
| 30 |
+
"allocated_bytes_peak_delta": 245760,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 123820125184,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 123820125184,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1888957952,
|
| 38 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 131522887680,
|
| 46 |
+
"reserved_bytes_peak": 131522887680,
|
| 47 |
+
"segment_count_current": 1946,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:14.507979Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2090284544,
|
| 55 |
+
"inactive_split_bytes_current_delta": 201326592,
|
| 56 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131522887680,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 131522887680,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1946,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:15.032068Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:77c8a786da42b11123c1b507ccba8d75475de0444ac2fb51ab183be836c33169
|
| 3 |
+
size 2585033
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 5444898816,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1492329984,
|
| 13 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 144529424384,
|
| 21 |
+
"reserved_bytes_peak": 144529424384,
|
| 22 |
+
"segment_count_current": 1800,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:17.851038Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 9080832,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 125632310272,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 123820370944,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 2090284544,
|
| 38 |
+
"inactive_split_bytes_peak": 11855969792,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 131522887680,
|
| 46 |
+
"reserved_bytes_peak": 131522887680,
|
| 47 |
+
"segment_count_current": 1946,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:15.036850Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1492329984,
|
| 55 |
+
"inactive_split_bytes_current_delta": -597954560,
|
| 56 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 866884096,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144529424384,
|
| 68 |
+
"reserved_bytes_current_delta": 13006536704,
|
| 69 |
+
"reserved_bytes_peak": 144529424384,
|
| 70 |
+
"reserved_bytes_peak_delta": 13006536704,
|
| 71 |
+
"segment_count_current": 1800,
|
| 72 |
+
"segment_count_current_delta": -146,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:17.903219Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9b33652e69f63d473cb99099831160e6df62d171d17ea04635b46203e741ce22
|
| 3 |
+
size 3223709
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1492329984,
|
| 13 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 144529424384,
|
| 21 |
+
"reserved_bytes_peak": 144529424384,
|
| 22 |
+
"segment_count_current": 1800,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.232044Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312985600,
|
| 33 |
+
"active_bytes_peak": 131077209088,
|
| 34 |
+
"allocated_bytes_current": 85312985600,
|
| 35 |
+
"allocated_bytes_peak": 123829451776,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1492329984,
|
| 38 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 144529424384,
|
| 46 |
+
"reserved_bytes_peak": 144529424384,
|
| 47 |
+
"segment_count_current": 1800,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:17.908736Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1492329984,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144529424384,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144529424384,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1800,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.241537Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e1b6f15dd911e1707859a136f7f892cfba0dc62d336b33dfbe5d493d67b45024
|
| 3 |
+
size 490897
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1492329984,
|
| 13 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 144529424384,
|
| 21 |
+
"reserved_bytes_peak": 144529424384,
|
| 22 |
+
"segment_count_current": 1800,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.558860Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312985600,
|
| 33 |
+
"active_bytes_peak": 131077209088,
|
| 34 |
+
"allocated_bytes_current": 85312985600,
|
| 35 |
+
"allocated_bytes_peak": 123829451776,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1492329984,
|
| 38 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 144529424384,
|
| 46 |
+
"reserved_bytes_peak": 144529424384,
|
| 47 |
+
"segment_count_current": 1800,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:18.242651Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1492329984,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12722853888,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144529424384,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144529424384,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1800,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.568299Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d8c34ef05c755ee78cc4804916538de48a6e25f529d16b87cc01c4287c9701f
|
| 3 |
+
size 490897
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 40775332352,
|
| 4 |
+
"active_bytes_peak": 92952749568,
|
| 5 |
+
"active_bytes_peak_delta": 48406973952,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 92952749568,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 92952749568,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1507276288,
|
| 13 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 103574142976,
|
| 21 |
+
"reserved_bytes_peak": 103574142976,
|
| 22 |
+
"segment_count_current": 1763,
|
| 23 |
+
"segment_count_peak": 1763,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:13.625012Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 40775332352,
|
| 29 |
+
"allocated_bytes_peak": 92952749568,
|
| 30 |
+
"allocated_bytes_peak_delta": 48406973952,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520609792,
|
| 33 |
+
"active_bytes_peak": 44545775616,
|
| 34 |
+
"allocated_bytes_current": 44520609792,
|
| 35 |
+
"allocated_bytes_peak": 44545775616,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 421357568,
|
| 38 |
+
"inactive_split_bytes_peak": 1025337344,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:52:35.069348Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1507276288,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1085918720,
|
| 56 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10851603968,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 103574142976,
|
| 68 |
+
"reserved_bytes_current_delta": 55322869760,
|
| 69 |
+
"reserved_bytes_peak": 103574142976,
|
| 70 |
+
"reserved_bytes_peak_delta": 55322869760,
|
| 71 |
+
"segment_count_current": 1763,
|
| 72 |
+
"segment_count_current_delta": 1142,
|
| 73 |
+
"segment_count_peak": 1763,
|
| 74 |
+
"segment_count_peak_delta": 1142,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:13.677734Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:68bddf7f6beb5f3e910a6c5ed9b1ad93a8e578933d3237cae60f7cc12c5ebb39
|
| 3 |
+
size 3134754
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 123820370944,
|
| 5 |
+
"active_bytes_peak_delta": 30867621376,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 123820370944,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 123820370944,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1909929472,
|
| 13 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 131522887680,
|
| 21 |
+
"reserved_bytes_peak": 131522887680,
|
| 22 |
+
"segment_count_current": 1946,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.224057Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123820370944,
|
| 30 |
+
"allocated_bytes_peak_delta": 30867621376,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 92952749568,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 92952749568,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1507276288,
|
| 38 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 103574142976,
|
| 46 |
+
"reserved_bytes_peak": 103574142976,
|
| 47 |
+
"segment_count_current": 1763,
|
| 48 |
+
"segment_count_peak": 1763,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:13.684182Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1909929472,
|
| 55 |
+
"inactive_split_bytes_current_delta": 402653184,
|
| 56 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131522887680,
|
| 68 |
+
"reserved_bytes_current_delta": 27948744704,
|
| 69 |
+
"reserved_bytes_peak": 131522887680,
|
| 70 |
+
"reserved_bytes_peak_delta": 27948744704,
|
| 71 |
+
"segment_count_current": 1946,
|
| 72 |
+
"segment_count_current_delta": 183,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 183,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:14.575607Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6d97ef67c9a9ad7a90d17c5de5d8e06d020761c33578e2b3bf83fa036b6b8834
|
| 3 |
+
size 2573452
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85295942144,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 125632556032,
|
| 5 |
+
"active_bytes_peak_delta": 1812185088,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85295942144,
|
| 8 |
+
"active_bytes_peak": 125632556032,
|
| 9 |
+
"allocated_bytes_current": 85295942144,
|
| 10 |
+
"allocated_bytes_peak": 123820616704,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 2111256064,
|
| 13 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 131522887680,
|
| 21 |
+
"reserved_bytes_peak": 131522887680,
|
| 22 |
+
"segment_count_current": 1946,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:14.987399Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85295942144,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123820616704,
|
| 30 |
+
"allocated_bytes_peak_delta": 245760,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 123820370944,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 123820370944,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1909929472,
|
| 38 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 131522887680,
|
| 46 |
+
"reserved_bytes_peak": 131522887680,
|
| 47 |
+
"segment_count_current": 1946,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:14.584007Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2111256064,
|
| 55 |
+
"inactive_split_bytes_current_delta": 201326592,
|
| 56 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 131522887680,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 131522887680,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1946,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:15.035788Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a88ee4bc057bc19463a395ba10d74e677ea6c279c19f5c5d2f0f288ab9a51376
|
| 3 |
+
size 2584936
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 5444653056,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1513301504,
|
| 13 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 144550395904,
|
| 21 |
+
"reserved_bytes_peak": 144550395904,
|
| 22 |
+
"segment_count_current": 1801,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:17.851160Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 8835072,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85295942144,
|
| 33 |
+
"active_bytes_peak": 125632556032,
|
| 34 |
+
"allocated_bytes_current": 85295942144,
|
| 35 |
+
"allocated_bytes_peak": 123820616704,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 2111256064,
|
| 38 |
+
"inactive_split_bytes_peak": 11876941312,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 131522887680,
|
| 46 |
+
"reserved_bytes_peak": 131522887680,
|
| 47 |
+
"segment_count_current": 1946,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:15.041759Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1513301504,
|
| 55 |
+
"inactive_split_bytes_current_delta": -597954560,
|
| 56 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 866884096,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144550395904,
|
| 68 |
+
"reserved_bytes_current_delta": 13027508224,
|
| 69 |
+
"reserved_bytes_peak": 144550395904,
|
| 70 |
+
"reserved_bytes_peak_delta": 13027508224,
|
| 71 |
+
"segment_count_current": 1801,
|
| 72 |
+
"segment_count_current_delta": -145,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:17.906426Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a2f46a2d722b8cf7e9a50687bfc66ebfde665b39e87bf808f629718274ae2da
|
| 3 |
+
size 3223619
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1513301504,
|
| 13 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 144550395904,
|
| 21 |
+
"reserved_bytes_peak": 144550395904,
|
| 22 |
+
"segment_count_current": 1801,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.232128Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312985600,
|
| 33 |
+
"active_bytes_peak": 131077209088,
|
| 34 |
+
"allocated_bytes_current": 85312985600,
|
| 35 |
+
"allocated_bytes_peak": 123829451776,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1513301504,
|
| 38 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 144550395904,
|
| 46 |
+
"reserved_bytes_peak": 144550395904,
|
| 47 |
+
"segment_count_current": 1801,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:17.912849Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1513301504,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144550395904,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144550395904,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1801,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.242759Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ab87d0e2e112218a3040ad8f4b67b6b14dfa6ee8b7e10f608e6b2221e049aca5
|
| 3 |
+
size 490865
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 85312985600,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 131077209088,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 85312985600,
|
| 8 |
+
"active_bytes_peak": 131077209088,
|
| 9 |
+
"allocated_bytes_current": 85312985600,
|
| 10 |
+
"allocated_bytes_peak": 123829451776,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1513301504,
|
| 13 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 144550395904,
|
| 21 |
+
"reserved_bytes_peak": 144550395904,
|
| 22 |
+
"segment_count_current": 1801,
|
| 23 |
+
"segment_count_peak": 1946,
|
| 24 |
+
"timestamp_utc": "2026-08-26T18:53:18.558939Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 85312985600,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 123829451776,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 85312985600,
|
| 33 |
+
"active_bytes_peak": 131077209088,
|
| 34 |
+
"allocated_bytes_current": 85312985600,
|
| 35 |
+
"allocated_bytes_peak": 123829451776,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1513301504,
|
| 38 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 144550395904,
|
| 46 |
+
"reserved_bytes_peak": 144550395904,
|
| 47 |
+
"segment_count_current": 1801,
|
| 48 |
+
"segment_count_peak": 1946,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:53:18.243882Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1513301504,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 12743825408,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 144550395904,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 144550395904,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1801,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1946,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T18:53:18.568826Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:15a5e7858f75d4d9bf75855daff66f2178ac6aefd791d0901817f2b27527f089
|
| 3 |
+
size 490865
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=QWEN3_235B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=better-memory-dump-support
|
| 6 |
+
commit=b7c52442
|
| 7 |
+
num_layers=12
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=12288
|
| 10 |
+
moe_ffn_hidden=1536
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=18874368
|
| 15 |
+
heads=64
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=12
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=0
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=65536
|
| 39 |
+
mean_tokens_per_expert=4096
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=16
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.25
|
| 44 |
+
cuda_graph=1
|
| 45 |
+
probe_mode=0
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=memory
|
| 48 |
+
memory_dump_mode=iteration
|
| 49 |
+
memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=6
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|