diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c5b89998d6cf1cf30f5b790b4e2974dfa41f7a9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85305508352, + "active_bytes_current_delta": 40784898560, + "active_bytes_peak": 92964658688, + "active_bytes_peak_delta": 48418883072, + "after": { + "active_bytes_current": 85305508352, + "active_bytes_peak": 92964658688, + "allocated_bytes_current": 85305508352, + "allocated_bytes_peak": 92964658688, + "device": 4, + "inactive_split_bytes_current": 1738882560, + "inactive_split_bytes_peak": 11517025280, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 103718846464, + "reserved_bytes_peak": 103718846464, + "segment_count_current": 1773, + "segment_count_peak": 1773, + "timestamp_utc": "2026-08-26T18:53:13.624966Z", + "world_size": 8 + }, + "allocated_bytes_current": 85305508352, + "allocated_bytes_current_delta": 40784898560, + "allocated_bytes_peak": 92964658688, + "allocated_bytes_peak_delta": 48418883072, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 4, + "inactive_split_bytes_current": 270362624, + "inactive_split_bytes_peak": 874342400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T18:52:35.069417Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1738882560, + "inactive_split_bytes_current_delta": 1468519936, + "inactive_split_bytes_peak": 11517025280, + "inactive_split_bytes_peak_delta": 10642682880, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 103718846464, + "reserved_bytes_current_delta": 55769563136, + "reserved_bytes_peak": 103718846464, + "reserved_bytes_peak_delta": 55769563136, + "segment_count_current": 1773, + "segment_count_current_delta": 1158, + "segment_count_peak": 1773, + "segment_count_peak_delta": 1158, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.678100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48e6da8fcffe89ab2b706effff9316c95538a65a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:694ea38ba10338075e96eeda159409303d08f01f3ce6b6542de81a88c56d0973 +size 3143234 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc23bfa5eaa496240a4a1fd99f4873f8083f53e8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85305508352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123848197632, + "active_bytes_peak_delta": 30883538944, + "after": { + "active_bytes_current": 85305508352, + "active_bytes_peak": 123848197632, + "allocated_bytes_current": 85305508352, + "allocated_bytes_peak": 123848197632, + "device": 4, + "inactive_split_bytes_current": 2143632896, + "inactive_split_bytes_peak": 11517025280, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 131409641472, + "reserved_bytes_peak": 131409641472, + "segment_count_current": 1969, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:14.223997Z", + "world_size": 8 + }, + "allocated_bytes_current": 85305508352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123848197632, + "allocated_bytes_peak_delta": 30883538944, + "before": { + "active_bytes_current": 85305508352, + "active_bytes_peak": 92964658688, + "allocated_bytes_current": 85305508352, + "allocated_bytes_peak": 92964658688, + "device": 4, + "inactive_split_bytes_current": 1738882560, + "inactive_split_bytes_peak": 11517025280, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 103718846464, + "reserved_bytes_peak": 103718846464, + "segment_count_current": 1773, + "segment_count_peak": 1773, + "timestamp_utc": "2026-08-26T18:53:13.684294Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2143632896, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 11517025280, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 131409641472, + "reserved_bytes_current_delta": 27690795008, + "reserved_bytes_peak": 131409641472, + "reserved_bytes_peak_delta": 27690795008, + "segment_count_current": 1969, + "segment_count_current_delta": 196, + "segment_count_peak": 1969, + "segment_count_peak_delta": 196, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.273495Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..658e89725779743493101f7a9fca7060b970f1b5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:618c5c4132390d0279c26422aede52859c7e602d326c07c4f03d0c5c8b884014 +size 2702011 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff47e94a56f09885b82acd6c6f15e2e1bf663cba --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85305508352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125663821824, + "active_bytes_peak_delta": 1815624192, + "after": { + "active_bytes_current": 85305508352, + "active_bytes_peak": 125663821824, + "allocated_bytes_current": 85305508352, + "allocated_bytes_peak": 123851882496, + "device": 4, + "inactive_split_bytes_current": 2344959488, + "inactive_split_bytes_peak": 11517025280, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 131409641472, + "reserved_bytes_peak": 131409641472, + "segment_count_current": 1969, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:14.987363Z", + "world_size": 8 + }, + "allocated_bytes_current": 85305508352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123851882496, + "allocated_bytes_peak_delta": 3684864, + "before": { + "active_bytes_current": 85305508352, + "active_bytes_peak": 123848197632, + "allocated_bytes_current": 85305508352, + "allocated_bytes_peak": 123848197632, + "device": 4, + "inactive_split_bytes_current": 2143632896, + "inactive_split_bytes_peak": 11517025280, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 131409641472, + "reserved_bytes_peak": 131409641472, + "segment_count_current": 1969, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:14.279853Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2344959488, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 11517025280, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 131409641472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131409641472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1969, + "segment_count_current_delta": 0, + "segment_count_peak": 1969, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.035956Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c36def44f050741fcc32910bbb346ff4d3eb2cd2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3612085b1d534624bdcdb31884a77147d8b8f9b10729fd61ccaa8514993da8f4 +size 2662107 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.json new file mode 100644 index 0000000000000000000000000000000000000000..056c73984b5e11d41da9f2f7f39346a98845c7e4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85322551808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131086775296, + "active_bytes_peak_delta": 5422953472, + "after": { + "active_bytes_current": 85322551808, + "active_bytes_peak": 131086775296, + "allocated_bytes_current": 85322551808, + "allocated_bytes_peak": 123851882496, + "device": 4, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_peak": 12959952384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 144793665536, + "reserved_bytes_peak": 144793665536, + "segment_count_current": 1829, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:17.851076Z", + "world_size": 8 + }, + "allocated_bytes_current": 85322551808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123851882496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85305508352, + "active_bytes_peak": 125663821824, + "allocated_bytes_current": 85305508352, + "allocated_bytes_peak": 123851882496, + "device": 4, + "inactive_split_bytes_current": 2344959488, + "inactive_split_bytes_peak": 11517025280, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 131409641472, + "reserved_bytes_peak": 131409641472, + "segment_count_current": 1969, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:15.042065Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_current_delta": -597954560, + "inactive_split_bytes_peak": 12959952384, + "inactive_split_bytes_peak_delta": 1442927104, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 144793665536, + "reserved_bytes_current_delta": 13384024064, + "reserved_bytes_peak": 144793665536, + "reserved_bytes_peak_delta": 13384024064, + "segment_count_current": 1829, + "segment_count_current_delta": -140, + "segment_count_peak": 1969, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.905515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0cc684a29820c82e152a1634f32134e079311c5d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd0d594d5d48552cf0c14b59f08361da894323b9a6a8a39e8e098115e24a667 +size 3557570 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3c1768332f703850e158d45d7f55c031f539b2b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85322551808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131086775296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85322551808, + "active_bytes_peak": 131086775296, + "allocated_bytes_current": 85322551808, + "allocated_bytes_peak": 123851882496, + "device": 4, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_peak": 12959952384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 144793665536, + "reserved_bytes_peak": 144793665536, + "segment_count_current": 1829, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:18.232062Z", + "world_size": 8 + }, + "allocated_bytes_current": 85322551808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123851882496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85322551808, + "active_bytes_peak": 131086775296, + "allocated_bytes_current": 85322551808, + "allocated_bytes_peak": 123851882496, + "device": 4, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_peak": 12959952384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 144793665536, + "reserved_bytes_peak": 144793665536, + "segment_count_current": 1829, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:17.912423Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12959952384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 144793665536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144793665536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1829, + "segment_count_current_delta": 0, + "segment_count_peak": 1969, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.242281Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba67e6c6ef01fa553ae75647f9a0f0a5326a0b38 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5e8a8bb0df40c64891d9ae6bfa2582ffdbc6a8ae7563522847f1564a50bb1ea +size 500333 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c4af726244f6122ebc2579ec29d4a7b2154d2f6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85322551808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131086775296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85322551808, + "active_bytes_peak": 131086775296, + "allocated_bytes_current": 85322551808, + "allocated_bytes_peak": 123851882496, + "device": 4, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_peak": 12959952384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 144793665536, + "reserved_bytes_peak": 144793665536, + "segment_count_current": 1829, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:18.558867Z", + "world_size": 8 + }, + "allocated_bytes_current": 85322551808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123851882496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85322551808, + "active_bytes_peak": 131086775296, + "allocated_bytes_current": 85322551808, + "allocated_bytes_peak": 123851882496, + "device": 4, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_peak": 12959952384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 144793665536, + "reserved_bytes_peak": 144793665536, + "segment_count_current": 1829, + "segment_count_peak": 1969, + "timestamp_utc": "2026-08-26T18:53:18.243412Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1747004928, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12959952384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 144793665536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144793665536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1829, + "segment_count_current_delta": 0, + "segment_count_peak": 1969, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.568683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58555e90207c658e37f1056da4fa34fda1118579 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24e13dc2e7201bffa77a91b8f62cd8b56d6a5fe9d5ddcc5ce109389363fdfdcd +size 500333 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c59734db66285bdd5c52fdfeafc493585f397a7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 40775332352, + "active_bytes_peak": 92952749568, + "active_bytes_peak_delta": 48406973952, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 5, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.624980Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 40775332352, + "allocated_bytes_peak": 92952749568, + "allocated_bytes_peak_delta": 48406973952, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 5, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:52:35.069371Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_current_delta": 1064947200, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 10830632448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 103574142976, + "reserved_bytes_current_delta": 55322869760, + "reserved_bytes_peak": 103574142976, + "reserved_bytes_peak_delta": 55322869760, + "segment_count_current": 1763, + "segment_count_current_delta": 1142, + "segment_count_peak": 1763, + "segment_count_peak_delta": 1142, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.676649Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87e0d93ba79133f4a02fd6c65110a1a60800fa40 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e646bda37ca523679dba662256b554f942923991e427b8372b97ef4530a348ce +size 3134744 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d351ab5662b3ea2112c5dc2291caac60968e674 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123820125184, + "active_bytes_peak_delta": 30867375616, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 5, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.223968Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820125184, + "allocated_bytes_peak_delta": 30867375616, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 5, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.682850Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 27948744704, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 27948744704, + "segment_count_current": 1946, + "segment_count_current_delta": 183, + "segment_count_peak": 1946, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.543073Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0221c1bd5521f42e97628a67dfcec254e3888b47 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8f153e6fb4a65e1f916bdba14421432711badae1a069b1a3a02f72c5409007c +size 2573434 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb0f6136141f4f817c93d2064b5025fb5eaa7ae1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125632310272, + "active_bytes_peak_delta": 1812185088, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632310272, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 5, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.987380Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820370944, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 5, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.550970Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1946, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.033243Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..55fe2cc11e4dac5c3958af7cc7a7e386e95f5cd1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31636bdd9a3a2ab584a046752635e338b392894ff07510af06acc175c3fe1dbd +size 2584909 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..304baa4c0e284931987b2c86508d8e28478fd9e3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 5444898816, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 5, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.851087Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 9080832, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632310272, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 5, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_peak": 11855969792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:15.038665Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": -597954560, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 866884096, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 13006536704, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 13006536704, + "segment_count_current": 1800, + "segment_count_current_delta": -146, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.904663Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d093a892987377b4bf6d081ad09212467aea419c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61489bf2e007af0e6fc131c0df3d99a67bef57dfe023553044ebeb9e86e24783 +size 3223552 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9689441f04bbfbeaf0f091f975e6d32e11a1e32 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 5, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.232117Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 5, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.910644Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.242332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc5e5cf692a59aca2b098c2852ebcbd6f86507a5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:421c3422903e64916555d3dda93bee8254beb070c1a142410b1a4c3f4d41dff7 +size 490746 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.json new file mode 100644 index 0000000000000000000000000000000000000000..059d5ce9cb1271af63a21db023e347e9751ba171 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 5, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.558884Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 5, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.243448Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.569017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ebb93f7fca8f79092ce9d8bd7eae19ee6f952400 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6839aa3dab10f50ef418ffab11d273249c24a192aca1ebce4e714816aac48090 +size 490746 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..239e089166efaa4f185b7a7c0688147fc5ab82b8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 40775332352, + "active_bytes_peak": 92952749568, + "active_bytes_peak_delta": 48406973952, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 6, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.624970Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 40775332352, + "allocated_bytes_peak": 92952749568, + "allocated_bytes_peak_delta": 48406973952, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 6, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:52:35.069260Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_current_delta": 1064947200, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 10830632448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 103574142976, + "reserved_bytes_current_delta": 55322869760, + "reserved_bytes_peak": 103574142976, + "reserved_bytes_peak_delta": 55322869760, + "segment_count_current": 1763, + "segment_count_current_delta": 1142, + "segment_count_peak": 1763, + "segment_count_peak_delta": 1142, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.677322Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20138cc8e292e365e7e3ec86d962268063cd240c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63fa1f05e4028667920d1b5bc237b672f59b4e6e98ddee231347d7614e526bc1 +size 3134747 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.json new file mode 100644 index 0000000000000000000000000000000000000000..302285feb5195880fb87db1410890ab8ac6186ce --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123820125184, + "active_bytes_peak_delta": 30867375616, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 6, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.223959Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820125184, + "allocated_bytes_peak_delta": 30867375616, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 6, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.683157Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 27948744704, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 27948744704, + "segment_count_current": 1946, + "segment_count_current_delta": 183, + "segment_count_peak": 1946, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.500901Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1191c5c58c978ca95dca2e9494e7f89d79dfb671 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031c2f06e89858824f1d85cc94f40af6b4435b68367a820ac81ea6dc14b0cef8 +size 2577463 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.json new file mode 100644 index 0000000000000000000000000000000000000000..decdcaa1d0a062c4f28a6fed7f2121a975068c0f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125632310272, + "active_bytes_peak_delta": 1812185088, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632310272, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 6, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.987309Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820370944, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 6, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.507979Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1946, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.032068Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6fa0bc5a558ee67d10d507e2b34133be7939393 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77c8a786da42b11123c1b507ccba8d75475de0444ac2fb51ab183be836c33169 +size 2585033 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2903d174e26dd03c1de0b3d7a50a99a3989fde1e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 5444898816, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 6, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.851038Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 9080832, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632310272, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 6, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_peak": 11855969792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:15.036850Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": -597954560, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 866884096, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 13006536704, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 13006536704, + "segment_count_current": 1800, + "segment_count_current_delta": -146, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.903219Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35656bd769ee89e911c005aecc82f6c8e890c3c2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b33652e69f63d473cb99099831160e6df62d171d17ea04635b46203e741ce22 +size 3223709 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.json new file mode 100644 index 0000000000000000000000000000000000000000..049b82b8adb21019d30c09c293fd40d80855e49a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 6, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.232044Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 6, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.908736Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.241537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9604228d6e55b3d9b80c344d7c24adbfb8427f47 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1b6f15dd911e1707859a136f7f892cfba0dc62d336b33dfbe5d493d67b45024 +size 490897 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..23ec742f6d3e0b3c54f0f2d63937ffb9e69d3253 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 6, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.558860Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 6, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.242651Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.568299Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9e89d596bbb6d145931447f4f0ce7838a434309 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d8c34ef05c755ee78cc4804916538de48a6e25f529d16b87cc01c4287c9701f +size 490897 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8694571d5bdbfc585734eca7a4f8a2328663cdcf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 40775332352, + "active_bytes_peak": 92952749568, + "active_bytes_peak_delta": 48406973952, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 7, + "inactive_split_bytes_current": 1507276288, + "inactive_split_bytes_peak": 11876941312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.625012Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 40775332352, + "allocated_bytes_peak": 92952749568, + "allocated_bytes_peak_delta": 48406973952, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 7, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:52:35.069348Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1507276288, + "inactive_split_bytes_current_delta": 1085918720, + "inactive_split_bytes_peak": 11876941312, + "inactive_split_bytes_peak_delta": 10851603968, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 103574142976, + "reserved_bytes_current_delta": 55322869760, + "reserved_bytes_peak": 103574142976, + "reserved_bytes_peak_delta": 55322869760, + "segment_count_current": 1763, + "segment_count_current_delta": 1142, + "segment_count_peak": 1763, + "segment_count_peak_delta": 1142, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.677734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f875a582ac25c2cf1f7462980264b8458b672c3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68bddf7f6beb5f3e910a6c5ed9b1ad93a8e578933d3237cae60f7cc12c5ebb39 +size 3134754 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4f37a2a0854dd1a0a46b73244eb3cc1275a75e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123820370944, + "active_bytes_peak_delta": 30867621376, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820370944, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 7, + "inactive_split_bytes_current": 1909929472, + "inactive_split_bytes_peak": 11876941312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.224057Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820370944, + "allocated_bytes_peak_delta": 30867621376, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 7, + "inactive_split_bytes_current": 1507276288, + "inactive_split_bytes_peak": 11876941312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.684182Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1909929472, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11876941312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 27948744704, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 27948744704, + "segment_count_current": 1946, + "segment_count_current_delta": 183, + "segment_count_peak": 1946, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.575607Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89a90850da68f677c3750c6bf560a2bcdbe8fa54 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d97ef67c9a9ad7a90d17c5de5d8e06d020761c33578e2b3bf83fa036b6b8834 +size 2573452 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd67da2ddbefa05a480ac5bdded0b10603c8ea95 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125632556032, + "active_bytes_peak_delta": 1812185088, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632556032, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820616704, + "device": 7, + "inactive_split_bytes_current": 2111256064, + "inactive_split_bytes_peak": 11876941312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.987399Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820616704, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820370944, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 7, + "inactive_split_bytes_current": 1909929472, + "inactive_split_bytes_peak": 11876941312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.584007Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2111256064, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 11876941312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1946, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.035788Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73dab9c19432f678da48cd76fcfa433405ee7b6e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a88ee4bc057bc19463a395ba10d74e677ea6c279c19f5c5d2f0f288ab9a51376 +size 2584936 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4bbc538f86a1e54e46e3b7fbd1c21dbc079a058 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 5444653056, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 7, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.851160Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 8835072, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632556032, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820616704, + "device": 7, + "inactive_split_bytes_current": 2111256064, + "inactive_split_bytes_peak": 11876941312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:15.041759Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_current_delta": -597954560, + "inactive_split_bytes_peak": 12743825408, + "inactive_split_bytes_peak_delta": 866884096, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 144550395904, + "reserved_bytes_current_delta": 13027508224, + "reserved_bytes_peak": 144550395904, + "reserved_bytes_peak_delta": 13027508224, + "segment_count_current": 1801, + "segment_count_current_delta": -145, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.906426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78c598c473c5e895aff1730fa34d06618700bbe4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a2f46a2d722b8cf7e9a50687bfc66ebfde665b39e87bf808f629718274ae2da +size 3223619 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f74c074e60be12319e41a7ffaf103482c19035b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 7, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.232128Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 7, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.912849Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12743825408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 144550395904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144550395904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1801, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.242759Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbf62c6f01d1956aabc7334ba01078653a777a9c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab87d0e2e112218a3040ad8f4b67b6b14dfa6ee8b7e10f608e6b2221e049aca5 +size 490865 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5670884efd64d536b8a433e12cb08690923903d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 7, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.558939Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 7, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.243882Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12743825408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 144550395904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144550395904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1801, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.568826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..672aa092d059eb7f60a7bbc02d3b7fd5ce2b594e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a5e7858f75d4d9bf75855daff66f2178ac6aefd791d0901817f2b27527f089 +size 490865 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..451c008e5346c3b5556204149555c180ec8330a2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e5807230d9dd2f35cfdf5f7bf9a8261256d21101 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log @@ -0,0 +1,1417 @@ +[18:52:09] WARN: better-memory-dump-support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 18:52:10.481000 570360 torch/distributed/run.py:766] +W0826 18:52:10.481000 570360 torch/distributed/run.py:766] ***************************************** +W0826 18:52:10.481000 570360 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:52:10.481000 570360 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 13526 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777 + + + + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 23 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 18:52:28.530333907 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:52:28.540027066 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:52:28.540072786 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:52:28.540102905 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:52:28.540130834 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:52:28.540130062 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:52:28.540175365 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 18:52:28.696335041 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.665 seconds +time to initialize megatron (seconds): 6.789 +[after megatron is initialized] datetime: 2026-08-26 18:52:34 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:52:34 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 18:52:34 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (619.97, 627.10) + train/valid/test-data-iterators-setup ..........: (60.30, 72.11) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 18:52:35 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 18:53:13] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 38552.1 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.778 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81341.68408203125 | max allocated: 88645.78466796875 | reserved: 98734.0 | max reserved: 98734.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 18:53:14] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 600.6 | throughput per GPU (TFLOP/s/GPU): 301.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:53:14] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 763.3 | throughput per GPU (TFLOP/s/GPU): 236.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.694370E+00 | loss scale: 1.0 | grad norm: 66.512 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 18:53:17] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2863.6 | throughput per GPU (TFLOP/s/GPU): 63.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338531E+01 | loss scale: 1.0 | grad norm: 19.942 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:53:18] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 381.0 | throughput per GPU (TFLOP/s/GPU): 474.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.009143E+01 | loss scale: 1.0 | grad norm: 83.318 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:53:18] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 326.7 | throughput per GPU (TFLOP/s/GPU): 553.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.363092E+01 | loss scale: 1.0 | grad norm: 121.752 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:53:18 +W0826 18:53:19.171000 570360 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:53:19.172000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570452 closing signal SIGTERM +W0826 18:53:19.174000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570453 closing signal SIGTERM +W0826 18:53:19.176000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570454 closing signal SIGTERM +W0826 18:53:19.178000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570455 closing signal SIGTERM +W0826 18:53:19.181000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570456 closing signal SIGTERM +W0826 18:53:19.185000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570457 closing signal SIGTERM +W0826 18:53:19.187000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570458 closing signal SIGTERM +W0826 18:53:19.189000 570360 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 570459 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 570360 got signal: 15 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..d2456a747d1185206170708bfaf4154668efd2f9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 629 +0, 827 +1, 827 +2, 829 +3, 827 +4, 831 +5, 831 +6, 837 +7, 835 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 47803 +1, 47803 +2, 47803 +3, 47803 +4, 47515 +5, 47803 +6, 47803 +7, 47803 +0, 49279 +1, 49279 +2, 49279 +3, 49279 +4, 48991 +5, 49279 +6, 49279 +7, 49279 +0, 49927 +1, 49927 +2, 49927 +3, 49927 +4, 49639 +5, 49927 +6, 49927 +7, 49927 +0, 53697 +1, 53697 +2, 53697 +3, 53697 +4, 53409 +5, 53697 +6, 53697 +7, 53697 +0, 54105 +1, 54105 +2, 54105 +3, 54105 +4, 53817 +5, 54105 +6, 54105 +7, 54105 +0, 54555 +1, 54555 +2, 54555 +3, 54555 +4, 54267 +5, 54555 +6, 54555 +7, 54555 +0, 55543 +1, 55543 +2, 55543 +3, 55543 +4, 55255 +5, 55543 +6, 55543 +7, 55543 +0, 56063 +1, 56065 +2, 56065 +3, 56067 +4, 55775 +5, 56065 +6, 56063 +7, 56063 +0, 57015 +1, 57019 +2, 57019 +3, 57023 +4, 56727 +5, 57019 +6, 57015 +7, 57015 +0, 57779 +1, 57785 +2, 57785 +3, 57791 +4, 57491 +5, 57785 +6, 57779 +7, 57779 +0, 63211 +1, 63191 +2, 63083 +3, 63045 +4, 62997 +5, 62245 +6, 63067 +7, 63211 +0, 96623 +1, 96615 +2, 96643 +3, 96663 +4, 96633 +5, 96627 +6, 96645 +7, 96663 +0, 97203 +1, 97185 +2, 97207 +3, 97211 +4, 97197 +5, 97187 +6, 97199 +7, 97197 +0, 97203 +1, 97185 +2, 97207 +3, 97211 +4, 97197 +5, 97187 +6, 97199 +7, 97197 +0, 97203 +1, 97185 +2, 97207 +3, 97211 +4, 97197 +5, 97187 +6, 97199 +7, 97197 +0, 97203 +1, 97187 +2, 97207 +3, 97211 +4, 97197 +5, 97187 +6, 97199 +7, 97197 +0, 97583 +1, 97483 +2, 97537 +3, 97539 +4, 97513 +5, 97961 +6, 97563 +7, 97539 +0, 99635 +1, 98835 +2, 100027 +3, 99669 +4, 100017 +5, 99615 +6, 98873 +7, 99643 +0, 105861 +1, 105101 +2, 106085 +3, 105597 +4, 106451 +5, 105923 +6, 105045 +7, 105867 +0, 130657 +1, 130073 +2, 131187 +3, 130831 +4, 131245 +5, 130849 +6, 130047 +7, 130831 +0, 130657 +1, 130073 +2, 131187 +3, 130831 +4, 131245 +5, 130849 +6, 130047 +7, 130831 +0, 129409 +1, 128825 +2, 4902 +3, 129583 +4, 4902 +5, 129601 +6, 128799 +7, 129583 +0, 3363 +1, 3363 +2, 3619 +3, 129580 +4, 4643 +5, 3363 +6, 3363 +7, 3366 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185433339234Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185433339234Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6eb5ceb47ac2cc7fd060219ee2ccab7fceb6966a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185433339234Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295239168, + "active_bytes_current_delta": 40774629376, + "active_bytes_peak": 90885390848, + "active_bytes_peak_delta": 46339615232, + "after": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 90885390848, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 90885390848, + "device": 0, + "inactive_split_bytes_current": 1535242240, + "inactive_split_bytes_peak": 15440537088, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 100881399808, + "reserved_bytes_peak": 100881399808, + "segment_count_current": 1720, + "segment_count_peak": 1720, + "timestamp_utc": "2026-08-26T18:54:33.338766Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295239168, + "allocated_bytes_current_delta": 40774629376, + "allocated_bytes_peak": 90885390848, + "allocated_bytes_peak_delta": 46339615232, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 0, + "inactive_split_bytes_current": 446523392, + "inactive_split_bytes_peak": 1050503168, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:53:51.406621Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1535242240, + "inactive_split_bytes_current_delta": 1088718848, + "inactive_split_bytes_peak": 15440537088, + "inactive_split_bytes_peak_delta": 14390033920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 100881399808, + "reserved_bytes_current_delta": 52630126592, + "reserved_bytes_peak": 100881399808, + "reserved_bytes_peak_delta": 52630126592, + "segment_count_current": 1720, + "segment_count_current_delta": 1099, + "segment_count_peak": 1720, + "segment_count_peak_delta": 1099, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185433339234Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.391313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185433339234Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185433339234Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..08fc84f4e8f9541da28fb6a963b8cdf38be895ad --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185433339234Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d43156afc029943ecf5e66400f7b70d5f272fa65292300c04b32c2c23de6b73 +size 3087876 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185434077954Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185434077954Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3150cba109de3702bdea39d3cf3d5bae744a1b6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185434077954Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295239168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121734807552, + "active_bytes_peak_delta": 30849416704, + "after": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 121734807552, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 121734807552, + "device": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_peak": 15440537088, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_peak": 126464557056, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:54:34.077369Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295239168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121734807552, + "allocated_bytes_peak_delta": 30849416704, + "before": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 90885390848, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 90885390848, + "device": 0, + "inactive_split_bytes_current": 1535242240, + "inactive_split_bytes_peak": 15440537088, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 100881399808, + "reserved_bytes_peak": 100881399808, + "segment_count_current": 1720, + "segment_count_peak": 1720, + "timestamp_utc": "2026-08-26T18:54:33.397877Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_current_delta": 406847488, + "inactive_split_bytes_peak": 15440537088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_current_delta": 25583157248, + "reserved_bytes_peak": 126464557056, + "reserved_bytes_peak_delta": 25583157248, + "segment_count_current": 1923, + "segment_count_current_delta": 203, + "segment_count_peak": 1923, + "segment_count_peak_delta": 203, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185434077954Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.129919Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185434077954Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185434077954Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee07e492c36955cce067f0f05d30861308a7c599 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185434077954Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63778a43b2fce4fb2d304d5694b1d3d407b3769343217f68f6c15f2ef67cde80 +size 2583856 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185434600718Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185434600718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87a84a7053c3935ca4a98066094395e621aade82 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185434600718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295239168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121734807552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 121734807552, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 121734807552, + "device": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_peak": 15440537088, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_peak": 126464557056, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:54:34.600268Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295239168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121734807552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 121734807552, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 121734807552, + "device": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_peak": 15440537088, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_peak": 126464557056, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:54:34.136514Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15440537088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126464557056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1923, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185434600718Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.650249Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185434600718Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185434600718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab45898d8964c7e07533f0e0b11641bb70ed4a4e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185434600718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ce4d4bd3016fa3af71cbf331e6471d8c83452ec6119aaf54992b78ba7301659 +size 2579474 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185435103272Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185435103272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..48babde0ef8c68f0c22a3e7c610807cc550d704d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185435103272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295239168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121734807552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 121734807552, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 121734807552, + "device": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_peak": 15440537088, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_peak": 126464557056, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:54:35.102852Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295239168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121734807552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 121734807552, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 121734807552, + "device": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_peak": 15440537088, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_peak": 126464557056, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:54:34.656499Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15440537088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126464557056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1923, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185435103272Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.150397Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185435103272Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185435103272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4348a5f41153c7de13c2d576c707741e2bac0894 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185435103272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f592982f25aec57aa26007c106df23bb7986caf5ae771b4d90af4f75392fd3c +size 2579152 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185435615969Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185435615969Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9d462eaf09ee1bc1378131dacee7022f2888e1b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185435615969Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295239168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121734807552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 121734807552, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 121734807552, + "device": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_peak": 15440537088, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_peak": 126464557056, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:54:35.615534Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295239168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121734807552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85295239168, + "active_bytes_peak": 121734807552, + "allocated_bytes_current": 85295239168, + "allocated_bytes_peak": 121734807552, + "device": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_peak": 15440537088, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_peak": 126464557056, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:54:35.156513Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1942089728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15440537088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 126464557056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126464557056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1923, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185435615969Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.663235Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185435615969Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185435615969Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbef3945d13f71c5f9d38eb1a1afc58d676d4880 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185435615969Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87aa7371fdc738d6e2942c238f72c18c09ce930f3f6991dd4616c5285fba9197 +size 2579188 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185433339455Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185433339455Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5393adc4ab8f6d1afa7d35483a55a12faa32853b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185433339455Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294330880, + "active_bytes_current_delta": 40773721088, + "active_bytes_peak": 90888826368, + "active_bytes_peak_delta": 46343050752, + "after": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 90888826368, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 90888826368, + "device": 1, + "inactive_split_bytes_current": 1521470464, + "inactive_split_bytes_peak": 14597119488, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 100107550720, + "reserved_bytes_peak": 100107550720, + "segment_count_current": 1723, + "segment_count_peak": 1723, + "timestamp_utc": "2026-08-26T18:54:33.338901Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294330880, + "allocated_bytes_current_delta": 40773721088, + "allocated_bytes_peak": 90888826368, + "allocated_bytes_peak_delta": 46343050752, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 1, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:53:51.406588Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1521470464, + "inactive_split_bytes_current_delta": 1100112896, + "inactive_split_bytes_peak": 14597119488, + "inactive_split_bytes_peak_delta": 13571782144, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 100107550720, + "reserved_bytes_current_delta": 51856277504, + "reserved_bytes_peak": 100107550720, + "reserved_bytes_peak_delta": 51856277504, + "segment_count_current": 1723, + "segment_count_current_delta": 1102, + "segment_count_peak": 1723, + "segment_count_peak_delta": 1102, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185433339455Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.394846Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185433339455Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185433339455Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff68d66b2bdffd9df722db05089e74c5f63bb5f3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185433339455Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b52c4c378ebad58969cadcd77ea5d57ba07e63f4be6c5c7718369a39e240bf7 +size 3071566 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185434077640Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185434077640Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e95bf5e255e73774f6792a95cd6f9f07d1bb43b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185434077640Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294330880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121726299136, + "active_bytes_peak_delta": 30837472768, + "after": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 121726299136, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 121726299136, + "device": 1, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_peak": 14597119488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 125852188672, + "reserved_bytes_peak": 125852188672, + "segment_count_current": 1930, + "segment_count_peak": 1930, + "timestamp_utc": "2026-08-26T18:54:34.077144Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294330880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121726299136, + "allocated_bytes_peak_delta": 30837472768, + "before": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 90888826368, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 90888826368, + "device": 1, + "inactive_split_bytes_current": 1521470464, + "inactive_split_bytes_peak": 14597119488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 100107550720, + "reserved_bytes_peak": 100107550720, + "segment_count_current": 1723, + "segment_count_peak": 1723, + "timestamp_utc": "2026-08-26T18:54:33.401520Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 14597119488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 125852188672, + "reserved_bytes_current_delta": 25744637952, + "reserved_bytes_peak": 125852188672, + "reserved_bytes_peak_delta": 25744637952, + "segment_count_current": 1930, + "segment_count_current_delta": 207, + "segment_count_peak": 1930, + "segment_count_peak_delta": 207, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185434077640Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.131384Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185434077640Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185434077640Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bc5001dd623f68fa08347e9bd2ee081e0be20b5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185434077640Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6bfbce0c1a0e751223e1b0f7e31dcf9fec5a0487d8e4dd6594eebcd19a33298 +size 2577857 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185434600693Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185434600693Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f31760ac2a7785466f233e5c2ca5fb49bb92cfd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185434600693Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294330880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121726299136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 121726299136, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 121726299136, + "device": 1, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_peak": 14597119488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 125852188672, + "reserved_bytes_peak": 125852188672, + "segment_count_current": 1930, + "segment_count_peak": 1930, + "timestamp_utc": "2026-08-26T18:54:34.600242Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294330880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121726299136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 121726299136, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 121726299136, + "device": 1, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_peak": 14597119488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 125852188672, + "reserved_bytes_peak": 125852188672, + "segment_count_current": 1930, + "segment_count_peak": 1930, + "timestamp_utc": "2026-08-26T18:54:34.138391Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14597119488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 125852188672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125852188672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1930, + "segment_count_current_delta": 0, + "segment_count_peak": 1930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185434600693Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.649589Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185434600693Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185434600693Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8a18c1ba442721f90ee68ae5d536778377c2813 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185434600693Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f6409175e2ab68520cd40c88995f69f31e14de8b144b1f17fe223c746ee5219 +size 2572768 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185435103405Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185435103405Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f24e6e65da996d936af7a261166c5ffaf32dc67 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185435103405Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294330880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121726299136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 121726299136, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 121726299136, + "device": 1, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_peak": 14597119488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 125852188672, + "reserved_bytes_peak": 125852188672, + "segment_count_current": 1930, + "segment_count_peak": 1930, + "timestamp_utc": "2026-08-26T18:54:35.102935Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294330880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121726299136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 121726299136, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 121726299136, + "device": 1, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_peak": 14597119488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 125852188672, + "reserved_bytes_peak": 125852188672, + "segment_count_current": 1930, + "segment_count_peak": 1930, + "timestamp_utc": "2026-08-26T18:54:34.656004Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14597119488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 125852188672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125852188672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1930, + "segment_count_current_delta": 0, + "segment_count_peak": 1930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185435103405Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.151575Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185435103405Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185435103405Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3341c59caa4fed9d39f37b8421554cc1aabf4b4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185435103405Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5890ed0cbea76bdc539a8726b5481654f93544cc0ce1e9536d26337fa7b6948 +size 2572826 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185435616100Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185435616100Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f7799391edba5097ea88ffb2d8d1fce4e5dc7cc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185435616100Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294330880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121726299136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 121726299136, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 121726299136, + "device": 1, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_peak": 14597119488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 125852188672, + "reserved_bytes_peak": 125852188672, + "segment_count_current": 1930, + "segment_count_peak": 1930, + "timestamp_utc": "2026-08-26T18:54:35.615650Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294330880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121726299136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85294330880, + "active_bytes_peak": 121726299136, + "allocated_bytes_current": 85294330880, + "allocated_bytes_peak": 121726299136, + "device": 1, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_peak": 14597119488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 125852188672, + "reserved_bytes_peak": 125852188672, + "segment_count_current": 1930, + "segment_count_peak": 1930, + "timestamp_utc": "2026-08-26T18:54:35.157907Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1926220800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14597119488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 125852188672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125852188672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1930, + "segment_count_current_delta": 0, + "segment_count_peak": 1930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185435616100Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.663787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185435616100Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185435616100Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40d9e692141c455ca43b4ea5ab7270d3c2babcf1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185435616100Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14ef5c1a40502c1ae0f0045cf1519b70977d345fce1be6286d03a1b4047d744e +size 2572698 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185433339234Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185433339234Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3acdcdd4f61ff39148d984d7781afebcb2ddba52 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185433339234Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293186048, + "active_bytes_current_delta": 40772576256, + "active_bytes_peak": 90888666112, + "active_bytes_peak_delta": 46342890496, + "after": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 90888666112, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 90888666112, + "device": 2, + "inactive_split_bytes_current": 1512129536, + "inactive_split_bytes_peak": 15599512064, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 101187584000, + "reserved_bytes_peak": 101187584000, + "segment_count_current": 1712, + "segment_count_peak": 1712, + "timestamp_utc": "2026-08-26T18:54:33.338747Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293186048, + "allocated_bytes_current_delta": 40772576256, + "allocated_bytes_peak": 90888666112, + "allocated_bytes_peak_delta": 46342890496, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 2, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:53:51.406589Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1512129536, + "inactive_split_bytes_current_delta": 1090771968, + "inactive_split_bytes_peak": 15599512064, + "inactive_split_bytes_peak_delta": 14574174720, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 101187584000, + "reserved_bytes_current_delta": 52936310784, + "reserved_bytes_peak": 101187584000, + "reserved_bytes_peak_delta": 52936310784, + "segment_count_current": 1712, + "segment_count_current_delta": 1091, + "segment_count_peak": 1712, + "segment_count_peak_delta": 1091, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185433339234Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.391022Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185433339234Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185433339234Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64d67214c3bc6a52001480918ca2b76730604cc5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185433339234Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35bec8517989321dc148636dbb4200af271be477c5734625997a1b4d02cf0ab2 +size 3069790 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185434077432Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185434077432Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67d06f02217d7ee9196253d9753bf0d9bdf7feda --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185434077432Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293186048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121713434624, + "active_bytes_peak_delta": 30824768512, + "after": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 121713434624, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 121713434624, + "device": 2, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_peak": 15599512064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127020302336, + "reserved_bytes_peak": 127020302336, + "segment_count_current": 1906, + "segment_count_peak": 1906, + "timestamp_utc": "2026-08-26T18:54:34.076993Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293186048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121713434624, + "allocated_bytes_peak_delta": 30824768512, + "before": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 90888666112, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 90888666112, + "device": 2, + "inactive_split_bytes_current": 1512129536, + "inactive_split_bytes_peak": 15599512064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 101187584000, + "reserved_bytes_peak": 101187584000, + "segment_count_current": 1712, + "segment_count_peak": 1712, + "timestamp_utc": "2026-08-26T18:54:33.397643Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 15599512064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127020302336, + "reserved_bytes_current_delta": 25832718336, + "reserved_bytes_peak": 127020302336, + "reserved_bytes_peak_delta": 25832718336, + "segment_count_current": 1906, + "segment_count_current_delta": 194, + "segment_count_peak": 1906, + "segment_count_peak_delta": 194, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185434077432Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.127457Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185434077432Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185434077432Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0b1a59d48848a7bb39a19af04facf8ac4582113 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185434077432Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2589e1416b6741ab05f01765cf9d676e9f8d6567ff3dc65f924c7e17ad478bd +size 2573422 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185434600505Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185434600505Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99e67564a847e7cf0d595537881c59502e4d5423 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185434600505Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293186048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121724200960, + "active_bytes_peak_delta": 10766336, + "after": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 121724200960, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 121724200960, + "device": 2, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_peak": 15599512064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127020302336, + "reserved_bytes_peak": 127020302336, + "segment_count_current": 1906, + "segment_count_peak": 1906, + "timestamp_utc": "2026-08-26T18:54:34.600080Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293186048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121724200960, + "allocated_bytes_peak_delta": 10766336, + "before": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 121713434624, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 121713434624, + "device": 2, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_peak": 15599512064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127020302336, + "reserved_bytes_peak": 127020302336, + "segment_count_current": 1906, + "segment_count_peak": 1906, + "timestamp_utc": "2026-08-26T18:54:34.134024Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15599512064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127020302336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127020302336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1906, + "segment_count_current_delta": 0, + "segment_count_peak": 1906, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185434600505Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.647703Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185434600505Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185434600505Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bee14b5acd369bfa15ba59892089b3259b39d94f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185434600505Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc00b673c356fd11146e25f911ab3cb7e56901d7988d8a05f38ce69bb099d183 +size 2564447 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185435103222Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185435103222Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24932c1fbf120d403bc9f5815839fbdd163fbc71 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185435103222Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293186048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121724200960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 121724200960, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 121724200960, + "device": 2, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_peak": 15599512064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127020302336, + "reserved_bytes_peak": 127020302336, + "segment_count_current": 1906, + "segment_count_peak": 1906, + "timestamp_utc": "2026-08-26T18:54:35.102810Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293186048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121724200960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 121724200960, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 121724200960, + "device": 2, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_peak": 15599512064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127020302336, + "reserved_bytes_peak": 127020302336, + "segment_count_current": 1906, + "segment_count_peak": 1906, + "timestamp_utc": "2026-08-26T18:54:34.653679Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15599512064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127020302336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127020302336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1906, + "segment_count_current_delta": 0, + "segment_count_peak": 1906, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185435103222Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.150071Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185435103222Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185435103222Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4dabb4194eff911a95c574d9a5e5fb36b9ec334c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185435103222Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d810b44e221fa0ee7edc5c0ae9870989f1d97ed6f548e55bfa74ab712e63e66 +size 2564529 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185435615906Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185435615906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08c779347a9a5241a16fbceceb451e235d9a1c76 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185435615906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293186048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121724200960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 121724200960, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 121724200960, + "device": 2, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_peak": 15599512064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127020302336, + "reserved_bytes_peak": 127020302336, + "segment_count_current": 1906, + "segment_count_peak": 1906, + "timestamp_utc": "2026-08-26T18:54:35.615496Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293186048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121724200960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85293186048, + "active_bytes_peak": 121724200960, + "allocated_bytes_current": 85293186048, + "allocated_bytes_peak": 121724200960, + "device": 2, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_peak": 15599512064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127020302336, + "reserved_bytes_peak": 127020302336, + "segment_count_current": 1906, + "segment_count_peak": 1906, + "timestamp_utc": "2026-08-26T18:54:35.156133Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1916879872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15599512064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127020302336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127020302336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1906, + "segment_count_current_delta": 0, + "segment_count_peak": 1906, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185435615906Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.662047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185435615906Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185435615906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5260b70b0e07434295b3cc6b826e012995d03789 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185435615906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cd014e60be6c20b118eca398a085f5a9694a2ec15dcd3323899feef5f4fed1b +size 2564495 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185433339316Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185433339316Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c04688d77b98c97d070947203d4781439640f68 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185433339316Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85301626368, + "active_bytes_current_delta": 40781016576, + "active_bytes_peak": 90917097984, + "active_bytes_peak_delta": 46371322368, + "after": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 90917097984, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 90917097984, + "device": 3, + "inactive_split_bytes_current": 1382054400, + "inactive_split_bytes_peak": 13066868224, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 100638130176, + "reserved_bytes_peak": 100638130176, + "segment_count_current": 1709, + "segment_count_peak": 1709, + "timestamp_utc": "2026-08-26T18:54:33.338816Z", + "world_size": 8 + }, + "allocated_bytes_current": 85301626368, + "allocated_bytes_current_delta": 40781016576, + "allocated_bytes_peak": 90917097984, + "allocated_bytes_peak_delta": 46371322368, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 3, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:53:51.406524Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1382054400, + "inactive_split_bytes_current_delta": 960696832, + "inactive_split_bytes_peak": 13066868224, + "inactive_split_bytes_peak_delta": 12041530880, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 100638130176, + "reserved_bytes_current_delta": 52386856960, + "reserved_bytes_peak": 100638130176, + "reserved_bytes_peak_delta": 52386856960, + "segment_count_current": 1709, + "segment_count_current_delta": 1088, + "segment_count_peak": 1709, + "segment_count_peak_delta": 1088, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185433339316Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.393046Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185433339316Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185433339316Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ae17e08016b8c1ba6ada01295b89682c636e71f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185433339316Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693a62999325a98d86656d36db0961c45661a135bdbcda670b65cf50dee98eff +size 3069247 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185434077427Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185434077427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2757f98fa7eee8b963b014114e63358b1b7137e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185434077427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85301626368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121739529728, + "active_bytes_peak_delta": 30822431744, + "after": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 121739529728, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 121739529728, + "device": 3, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_peak": 13066868224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1926, + "segment_count_peak": 1926, + "timestamp_utc": "2026-08-26T18:54:34.076986Z", + "world_size": 8 + }, + "allocated_bytes_current": 85301626368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121739529728, + "allocated_bytes_peak_delta": 30822431744, + "before": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 90917097984, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 90917097984, + "device": 3, + "inactive_split_bytes_current": 1382054400, + "inactive_split_bytes_peak": 13066868224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 100638130176, + "reserved_bytes_peak": 100638130176, + "segment_count_current": 1709, + "segment_count_peak": 1709, + "timestamp_utc": "2026-08-26T18:54:33.399604Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 13066868224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 26008879104, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 26008879104, + "segment_count_current": 1926, + "segment_count_current_delta": 217, + "segment_count_peak": 1926, + "segment_count_peak_delta": 217, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185434077427Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.128154Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185434077427Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185434077427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..23079d3d9ca0cec6c773eb6190fd3043fa9a6497 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185434077427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73cabc55847f779bb55cb3165c3dd57316701045c24eb0014fdd363b971318e0 +size 2577143 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185434600500Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185434600500Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1cdebfc9e1e8c830e4bf60c6b30c52c51fb59d52 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185434600500Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85301626368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121739529728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 121739529728, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 121739529728, + "device": 3, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_peak": 13066868224, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1926, + "segment_count_peak": 1926, + "timestamp_utc": "2026-08-26T18:54:34.600078Z", + "world_size": 8 + }, + "allocated_bytes_current": 85301626368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121739529728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 121739529728, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 121739529728, + "device": 3, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_peak": 13066868224, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1926, + "segment_count_peak": 1926, + "timestamp_utc": "2026-08-26T18:54:34.134283Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13066868224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1926, + "segment_count_current_delta": 0, + "segment_count_peak": 1926, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185434600500Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.647342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185434600500Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185434600500Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11fdcdd44427d7a32769689f7c496e05da9621f9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185434600500Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9b0df8f63013354eb7077e84335a327c8faa4804735ae6979d026496853ceb1 +size 2573343 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185435103228Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185435103228Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5053337d4bc4d41609810b37aa5a1ed2b4caad14 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185435103228Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85301626368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121739529728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 121739529728, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 121739529728, + "device": 3, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_peak": 13066868224, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1926, + "segment_count_peak": 1926, + "timestamp_utc": "2026-08-26T18:54:35.102820Z", + "world_size": 8 + }, + "allocated_bytes_current": 85301626368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121739529728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 121739529728, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 121739529728, + "device": 3, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_peak": 13066868224, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1926, + "segment_count_peak": 1926, + "timestamp_utc": "2026-08-26T18:54:34.652978Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13066868224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1926, + "segment_count_current_delta": 0, + "segment_count_peak": 1926, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185435103228Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.148705Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185435103228Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185435103228Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f14534e2b2085697a6854b6e2e26269bb945355f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185435103228Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b26dfe5edecae57a56cf9fa9047f466dc01ab7f78d2ddc17866ba5e1393f2be +size 2573219 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185435615942Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185435615942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80b83dc0f009ae9eab5863581bd615cefec852ce --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185435615942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85301626368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121739529728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 121739529728, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 121739529728, + "device": 3, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_peak": 13066868224, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1926, + "segment_count_peak": 1926, + "timestamp_utc": "2026-08-26T18:54:35.615518Z", + "world_size": 8 + }, + "allocated_bytes_current": 85301626368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121739529728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85301626368, + "active_bytes_peak": 121739529728, + "allocated_bytes_current": 85301626368, + "allocated_bytes_peak": 121739529728, + "device": 3, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_peak": 13066868224, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1926, + "segment_count_peak": 1926, + "timestamp_utc": "2026-08-26T18:54:35.154387Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1786804736, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13066868224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1926, + "segment_count_current_delta": 0, + "segment_count_peak": 1926, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185435615942Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.665083Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185435615942Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185435615942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a86c02d46c64f1dfde335cb9620ee9ef133c8e34 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185435615942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a8a8e8ae73b89201153400f2daddbc696e63a30179fc11d220f6468554bdb45 +size 2573185 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185433339775Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185433339775Z.json new file mode 100644 index 0000000000000000000000000000000000000000..534c2090f6ff18686b234200e0892de4489615f2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185433339775Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298810880, + "active_bytes_current_delta": 40778201088, + "active_bytes_peak": 90895610880, + "active_bytes_peak_delta": 46349835264, + "after": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 90895610880, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 90895610880, + "device": 4, + "inactive_split_bytes_current": 1942712320, + "inactive_split_bytes_peak": 13580800000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 101504253952, + "reserved_bytes_peak": 101504253952, + "segment_count_current": 1729, + "segment_count_peak": 1729, + "timestamp_utc": "2026-08-26T18:54:33.339015Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298810880, + "allocated_bytes_current_delta": 40778201088, + "allocated_bytes_peak": 90895610880, + "allocated_bytes_peak_delta": 46349835264, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 4, + "inactive_split_bytes_current": 270362624, + "inactive_split_bytes_peak": 874342400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T18:53:51.406521Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1942712320, + "inactive_split_bytes_current_delta": 1672349696, + "inactive_split_bytes_peak": 13580800000, + "inactive_split_bytes_peak_delta": 12706457600, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 101504253952, + "reserved_bytes_current_delta": 53554970624, + "reserved_bytes_peak": 101504253952, + "reserved_bytes_peak_delta": 53554970624, + "segment_count_current": 1729, + "segment_count_current_delta": 1114, + "segment_count_peak": 1729, + "segment_count_peak_delta": 1114, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185433339775Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.393134Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185433339775Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185433339775Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b75473daf110b055cba54cbfe7b4cc42711e6a6a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185433339775Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9828a49c9ecb37dd42a27fb7ee1d54fa4beaab4b5fa0916cf9d9624bc3c248d +size 3079745 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185434077425Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185434077425Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74ce3fb34f66d1ed2ee5d33479a288e9af01d991 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185434077425Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298810880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121737838080, + "active_bytes_peak_delta": 30842227200, + "after": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 121737838080, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 121737838080, + "device": 4, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_peak": 13580800000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127081119744, + "reserved_bytes_peak": 127081119744, + "segment_count_current": 1912, + "segment_count_peak": 1912, + "timestamp_utc": "2026-08-26T18:54:34.077006Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298810880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121737838080, + "allocated_bytes_peak_delta": 30842227200, + "before": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 90895610880, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 90895610880, + "device": 4, + "inactive_split_bytes_current": 1942712320, + "inactive_split_bytes_peak": 13580800000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 101504253952, + "reserved_bytes_peak": 101504253952, + "segment_count_current": 1729, + "segment_count_peak": 1729, + "timestamp_utc": "2026-08-26T18:54:33.399053Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 13580800000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127081119744, + "reserved_bytes_current_delta": 25576865792, + "reserved_bytes_peak": 127081119744, + "reserved_bytes_peak_delta": 25576865792, + "segment_count_current": 1912, + "segment_count_current_delta": 183, + "segment_count_peak": 1912, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185434077425Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.125309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185434077425Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185434077425Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9722447984000b7a49b8606f6746d059f5d4de3b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185434077425Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73896a161aee9d0efc6c5d5f32ce64ac5aa3697402bfadcab58ec1102894be90 +size 2638671 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185434600493Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185434600493Z.json new file mode 100644 index 0000000000000000000000000000000000000000..11b44a22745b9cfcb78be466e7d5a2630e8cc9d7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185434600493Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298810880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121751941120, + "active_bytes_peak_delta": 14103040, + "after": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 121751941120, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 121751941120, + "device": 4, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_peak": 13580800000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127081119744, + "reserved_bytes_peak": 127081119744, + "segment_count_current": 1912, + "segment_count_peak": 1912, + "timestamp_utc": "2026-08-26T18:54:34.600087Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298810880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121751941120, + "allocated_bytes_peak_delta": 14103040, + "before": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 121737838080, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 121737838080, + "device": 4, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_peak": 13580800000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127081119744, + "reserved_bytes_peak": 127081119744, + "segment_count_current": 1912, + "segment_count_peak": 1912, + "timestamp_utc": "2026-08-26T18:54:34.130508Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13580800000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127081119744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127081119744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1912, + "segment_count_current_delta": 0, + "segment_count_peak": 1912, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185434600493Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.643610Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185434600493Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185434600493Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db3774a83ebddd30feaa05161dfe8a1dc21badce --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185434600493Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1843d2504c8ecc9fe53ee51e3fffb09c1865390d9dcfb9268c3e103df1b7439 +size 2529955 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185435103215Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185435103215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acad1521f9cf4edea1a925e80dc4fdeea3870992 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185435103215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298810880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121751941120, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 121751941120, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 121751941120, + "device": 4, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_peak": 13580800000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127081119744, + "reserved_bytes_peak": 127081119744, + "segment_count_current": 1912, + "segment_count_peak": 1912, + "timestamp_utc": "2026-08-26T18:54:35.102824Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298810880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121751941120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 121751941120, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 121751941120, + "device": 4, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_peak": 13580800000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127081119744, + "reserved_bytes_peak": 127081119744, + "segment_count_current": 1912, + "segment_count_peak": 1912, + "timestamp_utc": "2026-08-26T18:54:34.648298Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13580800000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127081119744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127081119744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1912, + "segment_count_current_delta": 0, + "segment_count_peak": 1912, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185435103215Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.145922Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185435103215Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185435103215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8229473b2b82ca90643b79830ba69c8f771c379 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185435103215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af0581036c97289a73bf81651b634f09d743effd3f0e99935789a7d33afa0256 +size 2530044 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185435615979Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185435615979Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3463bb6cb7570ed168b771dfedacce6a5fa8203e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185435615979Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298810880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121751941120, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 121751941120, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 121751941120, + "device": 4, + "inactive_split_bytes_current": 2349559808, + "inactive_split_bytes_peak": 13580800000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127081119744, + "reserved_bytes_peak": 127081119744, + "segment_count_current": 1912, + "segment_count_peak": 1912, + "timestamp_utc": "2026-08-26T18:54:35.615537Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298810880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121751941120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85298810880, + "active_bytes_peak": 121751941120, + "allocated_bytes_current": 85298810880, + "allocated_bytes_peak": 121751941120, + "device": 4, + "inactive_split_bytes_current": 2347462656, + "inactive_split_bytes_peak": 13580800000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127081119744, + "reserved_bytes_peak": 127081119744, + "segment_count_current": 1912, + "segment_count_peak": 1912, + "timestamp_utc": "2026-08-26T18:54:35.150608Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2349559808, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 13580800000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127081119744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127081119744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1912, + "segment_count_current_delta": 0, + "segment_count_peak": 1912, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185435615979Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.659642Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185435615979Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185435615979Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b37b924a94c6bc9be20f0f6f0265816e1198f9b0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185435615979Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c8ce04b413c770ef0c41f619e268efb1d619392f28013bf36036fba6736e3f3 +size 2530020 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185433339444Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185433339444Z.json new file mode 100644 index 0000000000000000000000000000000000000000..91008162151beb0eda178cccec90f15d05190f29 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185433339444Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85296140288, + "active_bytes_current_delta": 40775530496, + "active_bytes_peak": 90889528320, + "active_bytes_peak_delta": 46343752704, + "after": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 90889528320, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 90889528320, + "device": 5, + "inactive_split_bytes_current": 1542729728, + "inactive_split_bytes_peak": 14870036992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 100940120064, + "reserved_bytes_peak": 100940120064, + "segment_count_current": 1727, + "segment_count_peak": 1727, + "timestamp_utc": "2026-08-26T18:54:33.338939Z", + "world_size": 8 + }, + "allocated_bytes_current": 85296140288, + "allocated_bytes_current_delta": 40775530496, + "allocated_bytes_peak": 90889528320, + "allocated_bytes_peak_delta": 46343752704, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 5, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:53:51.406503Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1542729728, + "inactive_split_bytes_current_delta": 1121372160, + "inactive_split_bytes_peak": 14870036992, + "inactive_split_bytes_peak_delta": 13844699648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 100940120064, + "reserved_bytes_current_delta": 52688846848, + "reserved_bytes_peak": 100940120064, + "reserved_bytes_peak_delta": 52688846848, + "segment_count_current": 1727, + "segment_count_current_delta": 1106, + "segment_count_peak": 1727, + "segment_count_peak_delta": 1106, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185433339444Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.393011Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185433339444Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185433339444Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a70f4bf2b959325b2a29865cb7a3e7c5c21e024 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185433339444Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f5c2f27696614eaeb89a836e980bd67687158187d0e5090a1ac5a0496632914 +size 3072218 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185434077459Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185434077459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46665151c5aa98c0c20ec3b492739c4dc2bfb730 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185434077459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85296140288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121725708800, + "active_bytes_peak_delta": 30836180480, + "after": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 121725708800, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 121725708800, + "device": 5, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_peak": 14870036992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 126665883648, + "reserved_bytes_peak": 126665883648, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T18:54:34.077019Z", + "world_size": 8 + }, + "allocated_bytes_current": 85296140288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121725708800, + "allocated_bytes_peak_delta": 30836180480, + "before": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 90889528320, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 90889528320, + "device": 5, + "inactive_split_bytes_current": 1542729728, + "inactive_split_bytes_peak": 14870036992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 100940120064, + "reserved_bytes_peak": 100940120064, + "segment_count_current": 1727, + "segment_count_peak": 1727, + "timestamp_utc": "2026-08-26T18:54:33.399298Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 14870036992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 126665883648, + "reserved_bytes_current_delta": 25725763584, + "reserved_bytes_peak": 126665883648, + "reserved_bytes_peak_delta": 25725763584, + "segment_count_current": 1934, + "segment_count_current_delta": 207, + "segment_count_peak": 1934, + "segment_count_peak_delta": 207, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185434077459Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.127498Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185434077459Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185434077459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25725620e46902e4a6fa240f62cb33e71be6e694 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185434077459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56945fca391fc82b48ad82580dcff010c741986c603c1ba26fa4a371130a432e +size 2578016 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185434600459Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185434600459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4435e91a4ca26c78a199e1f13669e8cc90d4d2c8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185434600459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85296140288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121725708800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 121725708800, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 121725708800, + "device": 5, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_peak": 14870036992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 126665883648, + "reserved_bytes_peak": 126665883648, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T18:54:34.600040Z", + "world_size": 8 + }, + "allocated_bytes_current": 85296140288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121725708800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 121725708800, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 121725708800, + "device": 5, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_peak": 14870036992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 126665883648, + "reserved_bytes_peak": 126665883648, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T18:54:34.133489Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14870036992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 126665883648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126665883648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185434600459Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.646028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185434600459Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185434600459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3589ed0134261155aaa560c49a0d14634fb8377 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185434600459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2c3e073872251b3f79916c93b6ff23a366fae4a0090e73fe1b41a12790a543 +size 2574354 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185435103189Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185435103189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..399ad72967ecae81b6246be97d2865194bb2748c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185435103189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85296140288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121725708800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 121725708800, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 121725708800, + "device": 5, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_peak": 14870036992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 126665883648, + "reserved_bytes_peak": 126665883648, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T18:54:35.102783Z", + "world_size": 8 + }, + "allocated_bytes_current": 85296140288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121725708800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 121725708800, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 121725708800, + "device": 5, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_peak": 14870036992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 126665883648, + "reserved_bytes_peak": 126665883648, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T18:54:34.651684Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14870036992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 126665883648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126665883648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185435103189Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.148109Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185435103189Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185435103189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a4263d8aad1cefced304b938f121563926e2360 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185435103189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06a5387433eaffa077b0f24c97ebe78907888b7fe1206e9fc7d6e57e120b58c0 +size 2574415 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185435615888Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185435615888Z.json new file mode 100644 index 0000000000000000000000000000000000000000..213a5baecbf7078fc4cd2fa646fc744b4d51c24a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185435615888Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85296140288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121725708800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 121725708800, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 121725708800, + "device": 5, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_peak": 14870036992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 126665883648, + "reserved_bytes_peak": 126665883648, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T18:54:35.615476Z", + "world_size": 8 + }, + "allocated_bytes_current": 85296140288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121725708800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85296140288, + "active_bytes_peak": 121725708800, + "allocated_bytes_current": 85296140288, + "allocated_bytes_peak": 121725708800, + "device": 5, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_peak": 14870036992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 126665883648, + "reserved_bytes_peak": 126665883648, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T18:54:35.153611Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1947480064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14870036992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 126665883648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126665883648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185435615888Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.663425Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185435615888Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185435615888Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a8e2a2dbfbbd72e06c43f8d850d9eb9dd8ce484 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185435615888Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8c2e17dbb6d763e1ff00a12d58e3d4edd9fef6ebf8e8fb0218f1034aa06647 +size 2574378 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185433339222Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185433339222Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25baafbe573ce0a20792d01597b57850ee393d6a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185433339222Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298498048, + "active_bytes_current_delta": 40777888256, + "active_bytes_peak": 90894585344, + "active_bytes_peak_delta": 46348809728, + "after": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 90894585344, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 90894585344, + "device": 6, + "inactive_split_bytes_current": 1578120704, + "inactive_split_bytes_peak": 13138029568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 100046733312, + "reserved_bytes_peak": 100046733312, + "segment_count_current": 1711, + "segment_count_peak": 1711, + "timestamp_utc": "2026-08-26T18:54:33.338708Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298498048, + "allocated_bytes_current_delta": 40777888256, + "allocated_bytes_peak": 90894585344, + "allocated_bytes_peak_delta": 46348809728, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 6, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:53:51.406556Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1578120704, + "inactive_split_bytes_current_delta": 1156763136, + "inactive_split_bytes_peak": 13138029568, + "inactive_split_bytes_peak_delta": 12112692224, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 100046733312, + "reserved_bytes_current_delta": 51795460096, + "reserved_bytes_peak": 100046733312, + "reserved_bytes_peak_delta": 51795460096, + "segment_count_current": 1711, + "segment_count_current_delta": 1090, + "segment_count_peak": 1711, + "segment_count_peak_delta": 1090, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185433339222Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.394382Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185433339222Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185433339222Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06a037c18d796ae2798f87185e5f83b45ec72224 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185433339222Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b0fc0d92eedb62e20a7e94b150ccefb1ef68596dbbf58af95ee284bfbe8e092 +size 3069563 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185434077444Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185434077444Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa093a7abf9cbf7ff0331b9617fb28c3fd46a18d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185434077444Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298498048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733173248, + "active_bytes_peak_delta": 30838587904, + "after": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 121733173248, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 121733173248, + "device": 6, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_peak": 13138029568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125824925696, + "reserved_bytes_peak": 125824925696, + "segment_count_current": 1918, + "segment_count_peak": 1918, + "timestamp_utc": "2026-08-26T18:54:34.077011Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298498048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733173248, + "allocated_bytes_peak_delta": 30838587904, + "before": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 90894585344, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 90894585344, + "device": 6, + "inactive_split_bytes_current": 1578120704, + "inactive_split_bytes_peak": 13138029568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 100046733312, + "reserved_bytes_peak": 100046733312, + "segment_count_current": 1711, + "segment_count_peak": 1711, + "timestamp_utc": "2026-08-26T18:54:33.400957Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 13138029568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125824925696, + "reserved_bytes_current_delta": 25778192384, + "reserved_bytes_peak": 125824925696, + "reserved_bytes_peak_delta": 25778192384, + "segment_count_current": 1918, + "segment_count_current_delta": 207, + "segment_count_peak": 1918, + "segment_count_peak_delta": 207, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185434077444Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.128029Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185434077444Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185434077444Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0285a31905df99cb2a260c0e922ebaeb1d40087e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185434077444Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872225b5799bbf57b1b8f51df7d2e64333e6f6e7fc8baa76ed88073b12dfa2d6 +size 2575441 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185434600445Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185434600445Z.json new file mode 100644 index 0000000000000000000000000000000000000000..571fd20705b26128298a0427f4b218b3d293ef83 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185434600445Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298498048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733173248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 121733173248, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 121733173248, + "device": 6, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_peak": 13138029568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125824925696, + "reserved_bytes_peak": 125824925696, + "segment_count_current": 1918, + "segment_count_peak": 1918, + "timestamp_utc": "2026-08-26T18:54:34.600037Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298498048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733173248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 121733173248, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 121733173248, + "device": 6, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_peak": 13138029568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 125824925696, + "reserved_bytes_peak": 125824925696, + "segment_count_current": 1918, + "segment_count_peak": 1918, + "timestamp_utc": "2026-08-26T18:54:34.134261Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13138029568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125824925696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125824925696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1918, + "segment_count_current_delta": 0, + "segment_count_peak": 1918, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185434600445Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.647549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185434600445Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185434600445Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e03011a41e77c68a60152cd15eaa5a7b86235c18 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185434600445Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e0d292d5175ccf676ca8d7d926d9412c2c8d97c3597b3d06d693e14a572436 +size 2568506 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185435103243Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185435103243Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b6f2cdd099d5260f0e09caff6573c8c98daff58 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185435103243Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298498048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733173248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 121733173248, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 121733173248, + "device": 6, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_peak": 13138029568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125824925696, + "reserved_bytes_peak": 125824925696, + "segment_count_current": 1918, + "segment_count_peak": 1918, + "timestamp_utc": "2026-08-26T18:54:35.102835Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298498048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733173248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 121733173248, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 121733173248, + "device": 6, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_peak": 13138029568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 125824925696, + "reserved_bytes_peak": 125824925696, + "segment_count_current": 1918, + "segment_count_peak": 1918, + "timestamp_utc": "2026-08-26T18:54:34.653162Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13138029568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125824925696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125824925696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1918, + "segment_count_current_delta": 0, + "segment_count_peak": 1918, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185435103243Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.149225Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185435103243Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185435103243Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20c53132d2fb5e005c9464d53ad2bc50885f3750 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185435103243Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf1a3d7fc9763c85deb20860d74f6d41a51cf8733e1d0c2d09a87a1c892e2fb9 +size 2568564 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185435615926Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185435615926Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3627df5170da21f0a970d61a531ea33da59ff3a4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185435615926Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85298498048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733173248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 121733173248, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 121733173248, + "device": 6, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_peak": 13138029568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125824925696, + "reserved_bytes_peak": 125824925696, + "segment_count_current": 1918, + "segment_count_peak": 1918, + "timestamp_utc": "2026-08-26T18:54:35.615511Z", + "world_size": 8 + }, + "allocated_bytes_current": 85298498048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733173248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85298498048, + "active_bytes_peak": 121733173248, + "allocated_bytes_current": 85298498048, + "allocated_bytes_peak": 121733173248, + "device": 6, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_peak": 13138029568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 125824925696, + "reserved_bytes_peak": 125824925696, + "segment_count_current": 1918, + "segment_count_peak": 1918, + "timestamp_utc": "2026-08-26T18:54:35.154870Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1982871040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13138029568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125824925696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125824925696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1918, + "segment_count_current_delta": 0, + "segment_count_peak": 1918, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185435615926Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.661719Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185435615926Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185435615926Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7687094e27f193907ffdc04f663ff5376128ae85 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185435615926Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d600cd7ba3f34b661af47b3f3ff1e38e65c8817ea44813346536e57f6e04bc8 +size 2568536 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185433339234Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185433339234Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b7db3e43f9f004702028fcb25245027552c2d87 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185433339234Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297588736, + "active_bytes_current_delta": 40776978944, + "active_bytes_peak": 90882820608, + "active_bytes_peak_delta": 46337044992, + "after": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 90882820608, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 90882820608, + "device": 7, + "inactive_split_bytes_current": 1555961344, + "inactive_split_bytes_peak": 14312579072, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 100925440000, + "reserved_bytes_peak": 100925440000, + "segment_count_current": 1722, + "segment_count_peak": 1722, + "timestamp_utc": "2026-08-26T18:54:33.338727Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297588736, + "allocated_bytes_current_delta": 40776978944, + "allocated_bytes_peak": 90882820608, + "allocated_bytes_peak_delta": 46337044992, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 7, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:53:51.406502Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1555961344, + "inactive_split_bytes_current_delta": 1134603776, + "inactive_split_bytes_peak": 14312579072, + "inactive_split_bytes_peak_delta": 13287241728, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 100925440000, + "reserved_bytes_current_delta": 52674166784, + "reserved_bytes_peak": 100925440000, + "reserved_bytes_peak_delta": 52674166784, + "segment_count_current": 1722, + "segment_count_current_delta": 1101, + "segment_count_peak": 1722, + "segment_count_peak_delta": 1101, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185433339234Z.pickle", + "timestamp_utc": "2026-08-26T18:54:33.390810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185433339234Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185433339234Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..451e2cbfe497e08f64d3c813dfc6bcc5fa0000aa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185433339234Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cfee30bac68a51f7766c90eed786096314909ea1c04c68d17cef75f84407e8d +size 3071383 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185434077434Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185434077434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5628535d82566fc572d991e783c3bb65fc0b8a9f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185434077434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297588736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121700692992, + "active_bytes_peak_delta": 30817872384, + "after": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 121700692992, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 121700692992, + "device": 7, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_peak": 14312579072, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1919, + "segment_count_peak": 1919, + "timestamp_utc": "2026-08-26T18:54:34.076972Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297588736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121700692992, + "allocated_bytes_peak_delta": 30817872384, + "before": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 90882820608, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 90882820608, + "device": 7, + "inactive_split_bytes_current": 1555961344, + "inactive_split_bytes_peak": 14312579072, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 100925440000, + "reserved_bytes_peak": 100925440000, + "segment_count_current": 1722, + "segment_count_peak": 1722, + "timestamp_utc": "2026-08-26T18:54:33.397157Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 14312579072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 25721569280, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 25721569280, + "segment_count_current": 1919, + "segment_count_current_delta": 197, + "segment_count_peak": 1919, + "segment_count_peak_delta": 197, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185434077434Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.127558Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185434077434Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185434077434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae2b95ece4d0e661a7480e8b8294a6d3ddb40372 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185434077434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4427cf70b69e40a6bbdc75b67401f3ba9e86a1f509f84cfd39ca9e14fc6051c +size 2575590 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185434600507Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185434600507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46892b79f41922188c5824cfc2f837363c2d65db --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185434600507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297588736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121718393344, + "active_bytes_peak_delta": 17700352, + "after": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 121718393344, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 121718393344, + "device": 7, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_peak": 14312579072, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1919, + "segment_count_peak": 1919, + "timestamp_utc": "2026-08-26T18:54:34.600088Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297588736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121718393344, + "allocated_bytes_peak_delta": 17700352, + "before": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 121700692992, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 121700692992, + "device": 7, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_peak": 14312579072, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1919, + "segment_count_peak": 1919, + "timestamp_utc": "2026-08-26T18:54:34.133657Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14312579072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1919, + "segment_count_current_delta": 0, + "segment_count_peak": 1919, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185434600507Z.pickle", + "timestamp_utc": "2026-08-26T18:54:34.646747Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185434600507Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185434600507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67bb819186d50378a105cc78ca9896eb41ce2d80 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185434600507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef8acc6d37f3de6b7b8552f632346bc092196a8daf63df5e1e36fa8ef3d9d88b +size 2569974 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185435103267Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185435103267Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fecc6d3354e07afebdd6b815039e2fa2f76caf72 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185435103267Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297588736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121718393344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 121718393344, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 121718393344, + "device": 7, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_peak": 14312579072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1919, + "segment_count_peak": 1919, + "timestamp_utc": "2026-08-26T18:54:35.102841Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297588736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121718393344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 121718393344, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 121718393344, + "device": 7, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_peak": 14312579072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1919, + "segment_count_peak": 1919, + "timestamp_utc": "2026-08-26T18:54:34.652298Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14312579072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1919, + "segment_count_current_delta": 0, + "segment_count_peak": 1919, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185435103267Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.148841Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185435103267Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185435103267Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..425654d618699fea801f2d90b4c83457ff6a1b9e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185435103267Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:498b4b168f9aa362344f62e5b26de091faf5fe5d4a4f77afac5f4530a58743a8 +size 2569768 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185435615934Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185435615934Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e1106ae1a7357b2fbdfd9b667c16107d4c8eaed --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185435615934Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297588736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121718393344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 121718393344, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 121718393344, + "device": 7, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_peak": 14312579072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1919, + "segment_count_peak": 1919, + "timestamp_utc": "2026-08-26T18:54:35.615516Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297588736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121718393344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85297588736, + "active_bytes_peak": 121718393344, + "allocated_bytes_current": 85297588736, + "allocated_bytes_peak": 121718393344, + "device": 7, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_peak": 14312579072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 126647009280, + "reserved_bytes_peak": 126647009280, + "segment_count_current": 1919, + "segment_count_peak": 1919, + "timestamp_utc": "2026-08-26T18:54:35.154414Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1960711680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14312579072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 126647009280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 126647009280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1919, + "segment_count_current_delta": 0, + "segment_count_peak": 1919, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185435615934Z.pickle", + "timestamp_utc": "2026-08-26T18:54:35.661273Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185435615934Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185435615934Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74187fb4c656d5773d5022355ef0fdceddf6a2b4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185435615934Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f333e883793a174751f4247a493733f907dbbeda86cd5b69488bfaf1194ac105 +size 2569734 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..596a944830a28694d49654c29da4fe3dceb6ea36 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=0 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=dynamic +capacity=dynamic +cuda_graph=0 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0a6cf2d2c1796c56a33342751cad8723f3fe94c5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log @@ -0,0 +1,1457 @@ +[18:53:25] WARN: better-memory-dump-support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=dynamic cap=dynamic cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 18:53:26.730000 573773 torch/distributed/run.py:766] +W0826 18:53:26.730000 573773 torch/distributed/run.py:766] ***************************************** +W0826 18:53:26.730000 573773 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:53:26.730000 573773 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... False + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 051 is connected to is connected to 2 is connected to 77 is connected to 7 peer ranks. peer ranks. 7 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 77Expected number of connected peer ranks is : 7 + +7 + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 76 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to [Gloo] Rank is connected to 000 peer ranks. is connected to peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0 peer ranks. 0 +Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 18:53:45.106438018 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:53:45.106518537 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:53:45.106914059 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:53:45.106998091 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:53:45.107620759 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:53:45.111622673 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:53:45.111629755 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 18:53:45.271253040 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.433 seconds +time to initialize megatron (seconds): 6.775 +[after megatron is initialized] datetime: 2026-08-26 18:53:50 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:53:51 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 18:53:51 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (594.63, 621.57) + train/valid/test-data-iterators-setup ..........: (118.77, 131.25) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 18:53:51 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 18:54:33] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 41930.8 | throughput per GPU (TFLOP/s/GPU): 4.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.810 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.85 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81343.87890625 | max allocated: 86675.06298828125 | reserved: 96208.0 | max reserved: 96208.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 18:54:34] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 739.1 | throughput per GPU (TFLOP/s/GPU): 244.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.337 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:54:34] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 523.1 | throughput per GPU (TFLOP/s/GPU): 345.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.686170E+00 | loss scale: 1.0 | grad norm: 62.923 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:54:35] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 502.6 | throughput per GPU (TFLOP/s/GPU): 359.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338089E+01 | loss scale: 1.0 | grad norm: 20.298 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:54:35] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 512.6 | throughput per GPU (TFLOP/s/GPU): 352.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.048583E+01 | loss scale: 1.0 | grad norm: 95.093 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:54:35 +[rank0]:[W826 18:54:37.186470997 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab3830b9995b9d4458a360e2be98f7f26efe7837 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,264 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 3117 +1, 3217 +2, 3117 +3, 3137 +4, 3117 +5, 3217 +6, 3137 +7, 3257 +0, 44809 +1, 44827 +2, 44833 +3, 44805 +4, 44793 +5, 44859 +6, 44803 +7, 44829 +0, 45897 +1, 45899 +2, 45909 +3, 45871 +4, 45895 +5, 45909 +6, 45899 +7, 45909 +0, 46919 +1, 46921 +2, 46931 +3, 46893 +4, 46917 +5, 46931 +6, 46921 +7, 46931 +0, 47015 +1, 47017 +2, 47027 +3, 46989 +4, 47013 +5, 47027 +6, 47017 +7, 47027 +0, 47221 +1, 47233 +2, 47211 +3, 47187 +4, 47215 +5, 47223 +6, 47209 +7, 47223 +0, 47879 +1, 47881 +2, 47891 +3, 47853 +4, 47877 +5, 47891 +6, 47881 +7, 47891 +0, 48007 +1, 48009 +2, 48019 +3, 47981 +4, 48005 +5, 48019 +6, 48009 +7, 48019 +0, 48427 +1, 48429 +2, 48419 +3, 48401 +4, 48425 +5, 48419 +6, 48389 +7, 48419 +0, 48687 +1, 48687 +2, 48675 +3, 48661 +4, 48683 +5, 48675 +6, 48649 +7, 48675 +0, 86139 +1, 86277 +2, 86063 +3, 86133 +4, 86115 +5, 86103 +6, 83519 +7, 86125 +0, 92047 +1, 92069 +2, 92053 +3, 92061 +4, 92065 +5, 92053 +6, 92049 +7, 92053 +0, 92047 +1, 92069 +2, 92053 +3, 92061 +4, 92065 +5, 92053 +6, 92049 +7, 92053 +0, 92047 +1, 92069 +2, 92053 +3, 92061 +4, 92065 +5, 92053 +6, 92049 +7, 92053 +0, 92047 +1, 92069 +2, 92053 +3, 92061 +4, 92065 +5, 92053 +6, 92049 +7, 92053 +0, 92047 +1, 92069 +2, 92053 +3, 92061 +4, 92065 +5, 92053 +6, 92049 +7, 92053 +0, 93241 +1, 93071 +2, 92863 +3, 93063 +4, 92971 +5, 93055 +6, 92859 +7, 92959 +0, 93889 +1, 93719 +2, 93607 +3, 93903 +4, 93619 +5, 93703 +6, 93507 +7, 93607 +0, 94481 +1, 94393 +2, 94217 +3, 94537 +4, 94233 +5, 94321 +6, 94121 +7, 94197 +0, 124295 +1, 124309 +2, 124319 +3, 124305 +4, 123769 +5, 124319 +6, 124005 +7, 124319 +0, 93401 +1, 96305 +2, 96353 +3, 94023 +4, 97389 +5, 93013 +6, 93253 +7, 96783 +0, 142295 +1, 142697 +2, 142733 +3, 142917 +4, 140645 +5, 142725 +6, 141945 +7, 142533 +0, 142879 +1, 142801 +2, 142733 +3, 143021 +4, 142669 +5, 142829 +6, 143105 +7, 142637 +0, 142063 +1, 141985 +2, 141917 +3, 142205 +4, 141853 +5, 142013 +6, 142289 +7, 141821 +0, 905 +1, 906 +2, 906 +3, 1162 +4, 141853 +5, 2186 +6, 906 +7, 906 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185552196737Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185552196737Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98ec607209ec5553cfd9041e343bcd8cbb692043 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185552196737Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620597248, + "active_bytes_current_delta": 41068995072, + "active_bytes_peak": 90443533824, + "active_bytes_peak_delta": 46885640192, + "after": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 90443533824, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 90443533824, + "device": 0, + "inactive_split_bytes_current": 4827129856, + "inactive_split_bytes_peak": 14725688320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 91760885760, + "reserved_bytes_peak": 91760885760, + "segment_count_current": 2235, + "segment_count_peak": 2235, + "timestamp_utc": "2026-08-26T18:55:52.196210Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620597248, + "allocated_bytes_current_delta": 41068995072, + "allocated_bytes_peak": 90443533824, + "allocated_bytes_peak_delta": 46885640192, + "before": { + "active_bytes_current": 43551602176, + "active_bytes_peak": 43557893632, + "allocated_bytes_current": 43551602176, + "allocated_bytes_peak": 43557893632, + "device": 0, + "inactive_split_bytes_current": 1256147456, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T18:55:12.944179Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4827129856, + "inactive_split_bytes_current_delta": 3570982400, + "inactive_split_bytes_peak": 14725688320, + "inactive_split_bytes_peak_delta": 13112775680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 91760885760, + "reserved_bytes_current_delta": 46938456064, + "reserved_bytes_peak": 91760885760, + "reserved_bytes_peak_delta": 46938456064, + "segment_count_current": 2235, + "segment_count_current_delta": 1350, + "segment_count_peak": 2235, + "segment_count_peak_delta": 1350, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185552196737Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.388017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185552196737Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185552196737Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33c5f5ef561674bea5fada225be5c66d0495a568 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185552196737Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0286affa750f9b6c34554c78547a0d76adefee84b395a3ba22bf8473d3241e5 +size 9810046 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185553844099Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185553844099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6520e96be62c753ef6034dc56d479e05811aac6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185553844099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620597248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121495907328, + "active_bytes_peak_delta": 31052373504, + "after": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 121495907328, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 5032650752, + "inactive_split_bytes_peak": 14725688320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122828095488, + "reserved_bytes_peak": 122828095488, + "segment_count_current": 3014, + "segment_count_peak": 3014, + "timestamp_utc": "2026-08-26T18:55:53.843677Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620597248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121193917440, + "allocated_bytes_peak_delta": 30750383616, + "before": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 90443533824, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 90443533824, + "device": 0, + "inactive_split_bytes_current": 4827129856, + "inactive_split_bytes_peak": 14725688320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 91760885760, + "reserved_bytes_peak": 91760885760, + "segment_count_current": 2235, + "segment_count_peak": 2235, + "timestamp_utc": "2026-08-26T18:55:52.427568Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5032650752, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 14725688320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122828095488, + "reserved_bytes_current_delta": 31067209728, + "reserved_bytes_peak": 122828095488, + "reserved_bytes_peak_delta": 31067209728, + "segment_count_current": 3014, + "segment_count_current_delta": 779, + "segment_count_peak": 3014, + "segment_count_peak_delta": 779, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185553844099Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.037260Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185553844099Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185553844099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91ce21180c55d98d659c1b7312c9468e0f01ebd1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185553844099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:451509eaffcd2aa88d60a33ea0b91a881c99fa826c8a5719f02df047497addfc +size 10181089 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185555302828Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185555302828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..beabf2eeb22409620e912d8d3a46b1ba4248c24e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185555302828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620597248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121622883328, + "active_bytes_peak_delta": 126976000, + "after": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 121622883328, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 4931987456, + "inactive_split_bytes_peak": 14725688320, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122828095488, + "reserved_bytes_peak": 122828095488, + "segment_count_current": 3014, + "segment_count_peak": 3014, + "timestamp_utc": "2026-08-26T18:55:55.302401Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620597248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121193917440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 121495907328, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 5032650752, + "inactive_split_bytes_peak": 14725688320, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122828095488, + "reserved_bytes_peak": 122828095488, + "segment_count_current": 3014, + "segment_count_peak": 3014, + "timestamp_utc": "2026-08-26T18:55:54.064156Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4931987456, + "inactive_split_bytes_current_delta": -100663296, + "inactive_split_bytes_peak": 14725688320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122828095488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122828095488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3014, + "segment_count_current_delta": 0, + "segment_count_peak": 3014, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185555302828Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.500082Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185555302828Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185555302828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84894cb578df4e3753b1a16e96436389d2ddc28b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185555302828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d508b80003a6e605bda95b97e60bfe5f8cbd860fdde07402cd1eccd201047a69 +size 9692398 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185559304518Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185559304518Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27be3f6966002e8e5aa2f86b1215a24203b08b63 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185559304518Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637640704, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127821019648, + "active_bytes_peak_delta": 6198136320, + "after": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 127821019648, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_peak": 14725688320, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_peak": 142205779968, + "segment_count_current": 3419, + "segment_count_peak": 3419, + "timestamp_utc": "2026-08-26T18:55:59.303992Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637640704, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121193917440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 121622883328, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 4931987456, + "inactive_split_bytes_peak": 14725688320, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122828095488, + "reserved_bytes_peak": 122828095488, + "segment_count_current": 3014, + "segment_count_peak": 3014, + "timestamp_utc": "2026-08-26T18:55:55.526495Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_current_delta": -50597888, + "inactive_split_bytes_peak": 14725688320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_current_delta": 19377684480, + "reserved_bytes_peak": 142205779968, + "reserved_bytes_peak_delta": 19377684480, + "segment_count_current": 3419, + "segment_count_current_delta": 405, + "segment_count_peak": 3419, + "segment_count_peak_delta": 405, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185559304518Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.514307Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185559304518Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185559304518Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b300c3a2d616147ea9cd1904da0d4cbe9e9cc63 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185559304518Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc20aff5401a3cef7c2f5a0af31d2f8f194bb22f65c23afb20138fe0eb13c93 +size 11684850 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185559950916Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185559950916Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25c9dd473a541cc3432bbdc5a1fc07d297a8abc2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185559950916Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637640704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127821019648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 127821019648, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_peak": 14725688320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_peak": 142205779968, + "segment_count_current": 3419, + "segment_count_peak": 3419, + "timestamp_utc": "2026-08-26T18:55:59.950452Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637640704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121193917440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 127821019648, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_peak": 14725688320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_peak": 142205779968, + "segment_count_current": 3419, + "segment_count_peak": 3419, + "timestamp_utc": "2026-08-26T18:55:59.542896Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14725688320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142205779968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3419, + "segment_count_current_delta": 0, + "segment_count_peak": 3419, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185559950916Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.976226Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185559950916Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185559950916Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..817df1ac756fdb318792af5a79c0f885e20d0bca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185559950916Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31575c18671a480efed38deb8c6ec5a8fd8a3d99404f86a81fe060661d2f5cc2 +size 1045663 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185600373627Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185600373627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0eef49b5d7265cc7966a5cd0971bf055ffb54658 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185600373627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637640704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127821019648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 127821019648, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_peak": 14725688320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_peak": 142205779968, + "segment_count_current": 3419, + "segment_count_peak": 3419, + "timestamp_utc": "2026-08-26T18:56:00.373222Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637640704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121193917440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 127821019648, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 121193917440, + "device": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_peak": 14725688320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_peak": 142205779968, + "segment_count_current": 3419, + "segment_count_peak": 3419, + "timestamp_utc": "2026-08-26T18:55:59.977970Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4881389568, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14725688320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142205779968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142205779968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3419, + "segment_count_current_delta": 0, + "segment_count_peak": 3419, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185600373627Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.395748Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185600373627Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185600373627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c589adea9fa5f1121d03196d90e53fb780832fca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185600373627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:567709471da0095794c33b0abde9400b4083a5ce92ef401f81cac32664510760 +size 1045663 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185552196719Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185552196719Z.json new file mode 100644 index 0000000000000000000000000000000000000000..323ca9d571faa67cd567e0eaf3ae1c3950544641 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185552196719Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84634250240, + "active_bytes_current_delta": 41080905216, + "active_bytes_peak": 90450784768, + "active_bytes_peak_delta": 46891148288, + "after": { + "active_bytes_current": 84634250240, + "active_bytes_peak": 90450784768, + "allocated_bytes_current": 84634250240, + "allocated_bytes_peak": 90450784768, + "device": 1, + "inactive_split_bytes_current": 4834448384, + "inactive_split_bytes_peak": 14564428800, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91582627840, + "reserved_bytes_peak": 91582627840, + "segment_count_current": 2239, + "segment_count_peak": 2239, + "timestamp_utc": "2026-08-26T18:55:52.196186Z", + "world_size": 8 + }, + "allocated_bytes_current": 84634250240, + "allocated_bytes_current_delta": 41080905216, + "allocated_bytes_peak": 90450784768, + "allocated_bytes_peak_delta": 46891148288, + "before": { + "active_bytes_current": 43553345024, + "active_bytes_peak": 43559636480, + "allocated_bytes_current": 43553345024, + "allocated_bytes_peak": 43559636480, + "device": 1, + "inactive_split_bytes_current": 1254404608, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:55:12.944149Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4834448384, + "inactive_split_bytes_current_delta": 3580043776, + "inactive_split_bytes_peak": 14564428800, + "inactive_split_bytes_peak_delta": 12951516160, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91582627840, + "reserved_bytes_current_delta": 46758100992, + "reserved_bytes_peak": 91582627840, + "reserved_bytes_peak_delta": 46758100992, + "segment_count_current": 2239, + "segment_count_current_delta": 1349, + "segment_count_peak": 2239, + "segment_count_peak_delta": 1349, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185552196719Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.388859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185552196719Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185552196719Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a48f69b6d9cccb0b4b7b84277eb4e50c1185c3f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185552196719Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ac293422d7fa5b879cd1dfe71783080d7201f7a135930c7fa4c753885bb853 +size 9760547 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185553844080Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185553844080Z.json new file mode 100644 index 0000000000000000000000000000000000000000..41496f6e9ca7fcf64d57dfdcd3ac52e6e36458e0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185553844080Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84634250240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121635896832, + "active_bytes_peak_delta": 31185112064, + "after": { + "active_bytes_current": 84634250240, + "active_bytes_peak": 121635896832, + "allocated_bytes_current": 84634250240, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 5037872128, + "inactive_split_bytes_peak": 14564428800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122842775552, + "reserved_bytes_peak": 122842775552, + "segment_count_current": 3020, + "segment_count_peak": 3020, + "timestamp_utc": "2026-08-26T18:55:53.843655Z", + "world_size": 8 + }, + "allocated_bytes_current": 84634250240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121183030784, + "allocated_bytes_peak_delta": 30732246016, + "before": { + "active_bytes_current": 84634250240, + "active_bytes_peak": 90450784768, + "allocated_bytes_current": 84634250240, + "allocated_bytes_peak": 90450784768, + "device": 1, + "inactive_split_bytes_current": 4834448384, + "inactive_split_bytes_peak": 14564428800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 91582627840, + "reserved_bytes_peak": 91582627840, + "segment_count_current": 2239, + "segment_count_peak": 2239, + "timestamp_utc": "2026-08-26T18:55:52.428909Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5037872128, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 14564428800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122842775552, + "reserved_bytes_current_delta": 31260147712, + "reserved_bytes_peak": 122842775552, + "reserved_bytes_peak_delta": 31260147712, + "segment_count_current": 3020, + "segment_count_current_delta": 781, + "segment_count_peak": 3020, + "segment_count_peak_delta": 781, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185553844080Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.035377Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185553844080Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185553844080Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7340d1ae288e6775fa11727282d1a5a9f044908 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185553844080Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:944d706a18a6786748c01624eaff477c13671fdb65ac978b92d0aede1a197162 +size 10182886 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185555302841Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185555302841Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbd714c1d6e7ffc7da671d64d305255375bb3c19 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185555302841Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84634250240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121635896832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84634250240, + "active_bytes_peak": 121635896832, + "allocated_bytes_current": 84634250240, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 4836545536, + "inactive_split_bytes_peak": 14564428800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122842775552, + "reserved_bytes_peak": 122842775552, + "segment_count_current": 3020, + "segment_count_peak": 3020, + "timestamp_utc": "2026-08-26T18:55:55.302415Z", + "world_size": 8 + }, + "allocated_bytes_current": 84634250240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121183030784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84634250240, + "active_bytes_peak": 121635896832, + "allocated_bytes_current": 84634250240, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 5037872128, + "inactive_split_bytes_peak": 14564428800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122842775552, + "reserved_bytes_peak": 122842775552, + "segment_count_current": 3020, + "segment_count_peak": 3020, + "timestamp_utc": "2026-08-26T18:55:54.062520Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4836545536, + "inactive_split_bytes_current_delta": -201326592, + "inactive_split_bytes_peak": 14564428800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122842775552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122842775552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3020, + "segment_count_current_delta": 0, + "segment_count_peak": 3020, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185555302841Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.492679Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185555302841Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185555302841Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea66e19c37fe71e83c8ad85c838a718339c21aa3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185555302841Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18feecf650efc3da1109a8dd729039515cdddf073a4d5604fffe6bb3a6ddf4af +size 9694475 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185559304567Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185559304567Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0560a6149997a843754cc95bb361cfd2600f90e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185559304567Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651293696, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127834672640, + "active_bytes_peak_delta": 6198775808, + "after": { + "active_bytes_current": 84651293696, + "active_bytes_peak": 127834672640, + "allocated_bytes_current": 84651293696, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_peak": 14564428800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142123991040, + "reserved_bytes_peak": 142123991040, + "segment_count_current": 3422, + "segment_count_peak": 3422, + "timestamp_utc": "2026-08-26T18:55:59.304028Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651293696, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121183030784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84634250240, + "active_bytes_peak": 121635896832, + "allocated_bytes_current": 84634250240, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 4836545536, + "inactive_split_bytes_peak": 14564428800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122842775552, + "reserved_bytes_peak": 122842775552, + "segment_count_current": 3020, + "segment_count_peak": 3020, + "timestamp_utc": "2026-08-26T18:55:55.518679Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_current_delta": 52162560, + "inactive_split_bytes_peak": 14564428800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142123991040, + "reserved_bytes_current_delta": 19281215488, + "reserved_bytes_peak": 142123991040, + "reserved_bytes_peak_delta": 19281215488, + "segment_count_current": 3422, + "segment_count_current_delta": 402, + "segment_count_peak": 3422, + "segment_count_peak_delta": 402, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185559304567Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.512692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185559304567Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185559304567Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6076d42f74cca1f660c8bd0e793a450e851836ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185559304567Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c343315f0826053dbe6f5e20a16ab1a58710a55368a476d63b237aacab29df3e +size 13053931 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185559950897Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185559950897Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe91a76aab368b649e97bd66256c423d6ec3d80 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185559950897Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651293696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127834672640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84651293696, + "active_bytes_peak": 127834672640, + "allocated_bytes_current": 84651293696, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_peak": 14564428800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142123991040, + "reserved_bytes_peak": 142123991040, + "segment_count_current": 3422, + "segment_count_peak": 3422, + "timestamp_utc": "2026-08-26T18:55:59.950452Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651293696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121183030784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84651293696, + "active_bytes_peak": 127834672640, + "allocated_bytes_current": 84651293696, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_peak": 14564428800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142123991040, + "reserved_bytes_peak": 142123991040, + "segment_count_current": 3422, + "segment_count_peak": 3422, + "timestamp_utc": "2026-08-26T18:55:59.541361Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14564428800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142123991040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142123991040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3422, + "segment_count_current_delta": 0, + "segment_count_peak": 3422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185559950897Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.976477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185559950897Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185559950897Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..503d8b327aed03e4ebb715019461750c60b4dda8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185559950897Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acbdcc2961439fe8f416019773eb9a604621a2928ed40bcd2a27889a49b4d453 +size 1152544 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185600373598Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185600373598Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3aa2a17a32402dcb4181ca936cd0fce0c805c863 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185600373598Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651293696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127834672640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84651293696, + "active_bytes_peak": 127834672640, + "allocated_bytes_current": 84651293696, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_peak": 14564428800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142123991040, + "reserved_bytes_peak": 142123991040, + "segment_count_current": 3422, + "segment_count_peak": 3422, + "timestamp_utc": "2026-08-26T18:56:00.373196Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651293696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121183030784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84651293696, + "active_bytes_peak": 127834672640, + "allocated_bytes_current": 84651293696, + "allocated_bytes_peak": 121183030784, + "device": 1, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_peak": 14564428800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142123991040, + "reserved_bytes_peak": 142123991040, + "segment_count_current": 3422, + "segment_count_peak": 3422, + "timestamp_utc": "2026-08-26T18:55:59.978228Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4888708096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14564428800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142123991040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142123991040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3422, + "segment_count_current_delta": 0, + "segment_count_peak": 3422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185600373598Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.395694Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185600373598Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185600373598Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c13ab80eca727d7a74c479c94c0d1fa0b09c033b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185600373598Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80f80afc4fc9bdfdadb6ca00a7bb62a65cef37c71f8adc66a11b7fbdefe20ed1 +size 1152544 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185552196663Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185552196663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a0e6ac36700dca6427974f13fa954fae434dc5f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185552196663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 41083820544, + "active_bytes_peak": 90446782464, + "active_bytes_peak_delta": 46886947840, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 90446782464, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 90446782464, + "device": 2, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_peak": 14460597760, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 91465187328, + "reserved_bytes_peak": 91465187328, + "segment_count_current": 2223, + "segment_count_peak": 2223, + "timestamp_utc": "2026-08-26T18:55:52.196175Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 41083820544, + "allocated_bytes_peak": 90446782464, + "allocated_bytes_peak_delta": 46886947840, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 2, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:55:12.944178Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_current_delta": 3400967680, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 12847685120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 91465187328, + "reserved_bytes_current_delta": 46630174720, + "reserved_bytes_peak": 91465187328, + "reserved_bytes_peak_delta": 46630174720, + "segment_count_current": 2223, + "segment_count_current_delta": 1340, + "segment_count_peak": 2223, + "segment_count_peak_delta": 1340, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185552196663Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.382244Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185552196663Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185552196663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..577f0c81dee0bfad82a8e5384eaa3b4b6198cb49 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185552196663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9df2889d462d27f047f0fe2d06450a28edfa8ed8908b4bb37f5e94d557290779 +size 9739336 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185553844054Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185553844054Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc175c25a81ab77ad0965e3d42a5bdbd30dc496b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185553844054Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121666840064, + "active_bytes_peak_delta": 31220057600, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4762128896, + "inactive_split_bytes_peak": 14460597760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:53.843647Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 30767072768, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 90446782464, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 90446782464, + "device": 2, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_peak": 14460597760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 91465187328, + "reserved_bytes_peak": 91465187328, + "segment_count_current": 2223, + "segment_count_peak": 2223, + "timestamp_utc": "2026-08-26T18:55:52.411987Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4762128896, + "inactive_split_bytes_current_delta": 104857600, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122853261312, + "reserved_bytes_current_delta": 31388073984, + "reserved_bytes_peak": 122853261312, + "reserved_bytes_peak_delta": 31388073984, + "segment_count_current": 2996, + "segment_count_current_delta": 773, + "segment_count_peak": 2996, + "segment_count_peak_delta": 773, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185553844054Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.020545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185553844054Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185553844054Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..006807a7bb598b0774686e2333d3cd148cd31227 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185553844054Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93ab7d51a0137d62eebe2c5a4bb57f3db1a1c4ddbb7afdf77aa7fc50e3c1ff16 +size 10167963 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185555302790Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185555302790Z.json new file mode 100644 index 0000000000000000000000000000000000000000..886edc3d411c299a09acb16c566560ccacd1c328 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185555302790Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121666840064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4762128896, + "inactive_split_bytes_peak": 14460597760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:55.302384Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4762128896, + "inactive_split_bytes_peak": 14460597760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:54.040038Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4762128896, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122853261312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122853261312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2996, + "segment_count_current_delta": 0, + "segment_count_peak": 2996, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185555302790Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.485958Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185555302790Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185555302790Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8265279a99fff74718c4cf2b282431045266d5c3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185555302790Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0acc18eb8b9462a7a85b09e026a6638e20e0e8d8b2429b3485a2bdf88c868dc +size 9676900 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185559304475Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185559304475Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a15bcd9c58612a5e39568e9a1685d3e219b1d1b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185559304475Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 6170946048, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142052687872, + "reserved_bytes_peak": 142052687872, + "segment_count_current": 3397, + "segment_count_peak": 3397, + "timestamp_utc": "2026-08-26T18:55:59.303969Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4762128896, + "inactive_split_bytes_peak": 14460597760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:55.506341Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": -50597888, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142052687872, + "reserved_bytes_current_delta": 19199426560, + "reserved_bytes_peak": 142052687872, + "reserved_bytes_peak_delta": 19199426560, + "segment_count_current": 3397, + "segment_count_current_delta": 401, + "segment_count_peak": 3397, + "segment_count_peak_delta": 401, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185559304475Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.505115Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185559304475Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185559304475Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af9bc8a456f387f4b128da4dc6c67b3e5c7caf16 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185559304475Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:980578e52639a4788945a1ad6133039fdb7d12209531a159032b0ee5c07f7de2 +size 11672062 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185559950832Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185559950832Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8c05f6f06a260236b49e60540dfa674c3b4ee7c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185559950832Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142052687872, + "reserved_bytes_peak": 142052687872, + "segment_count_current": 3397, + "segment_count_peak": 3397, + "timestamp_utc": "2026-08-26T18:55:59.950412Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142052687872, + "reserved_bytes_peak": 142052687872, + "segment_count_current": 3397, + "segment_count_peak": 3397, + "timestamp_utc": "2026-08-26T18:55:59.527014Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142052687872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142052687872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3397, + "segment_count_current_delta": 0, + "segment_count_peak": 3397, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185559950832Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.973091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185559950832Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185559950832Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..501c998916f48805af3d51d030c4afdaecad2da6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185559950832Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22f676052260e985ff10f4ed3aa5bd0fe1a2534ab0816b0c1249659eb221328e +size 1031099 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185600373586Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185600373586Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c774c4fd9f76dbb0feaa75f80d83a4b1236bccb4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185600373586Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142052687872, + "reserved_bytes_peak": 142052687872, + "segment_count_current": 3397, + "segment_count_peak": 3397, + "timestamp_utc": "2026-08-26T18:56:00.373186Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 2, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142052687872, + "reserved_bytes_peak": 142052687872, + "segment_count_current": 3397, + "segment_count_peak": 3397, + "timestamp_utc": "2026-08-26T18:55:59.974792Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142052687872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142052687872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3397, + "segment_count_current_delta": 0, + "segment_count_peak": 3397, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185600373586Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.394426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185600373586Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185600373586Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8b4a3699bd52e6740277c99d8eaee4920075c3c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185600373586Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9b9f3aa91504fbc5e7b607f5eece5061f661c1a243f796a43cd0ef8b4c861aa +size 1031099 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185552196776Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185552196776Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac90f9a744cf20c71702d1f5f9ee4bd185535808 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185552196776Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84646538240, + "active_bytes_current_delta": 41089715712, + "active_bytes_peak": 90445420032, + "active_bytes_peak_delta": 46882306048, + "after": { + "active_bytes_current": 84646538240, + "active_bytes_peak": 90445420032, + "allocated_bytes_current": 84646538240, + "allocated_bytes_peak": 90445420032, + "device": 3, + "inactive_split_bytes_current": 4950086656, + "inactive_split_bytes_peak": 14758083584, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 91775565824, + "reserved_bytes_peak": 91775565824, + "segment_count_current": 2239, + "segment_count_peak": 2239, + "timestamp_utc": "2026-08-26T18:55:52.196216Z", + "world_size": 8 + }, + "allocated_bytes_current": 84646538240, + "allocated_bytes_current_delta": 41089715712, + "allocated_bytes_peak": 90445420032, + "allocated_bytes_peak_delta": 46882306048, + "before": { + "active_bytes_current": 43556822528, + "active_bytes_peak": 43563113984, + "allocated_bytes_current": 43556822528, + "allocated_bytes_peak": 43563113984, + "device": 3, + "inactive_split_bytes_current": 1223664128, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T18:55:12.944271Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4950086656, + "inactive_split_bytes_current_delta": 3726422528, + "inactive_split_bytes_peak": 14758083584, + "inactive_split_bytes_peak_delta": 13145170944, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 91775565824, + "reserved_bytes_current_delta": 46980399104, + "reserved_bytes_peak": 91775565824, + "reserved_bytes_peak_delta": 46980399104, + "segment_count_current": 2239, + "segment_count_current_delta": 1352, + "segment_count_peak": 2239, + "segment_count_peak_delta": 1352, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185552196776Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.385699Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185552196776Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185552196776Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..444cd226b1ae6924c193ce50aeb00bbd080c1158 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185552196776Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:815eeef3a5e4937ed6ee7ab6bf3d0bb4be4bf5bebbb9ac6ccb2fee07b6d46af2 +size 9760266 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185553844080Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185553844080Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61b9594a0ed80f580d682fa4f0a63966f6c644d0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185553844080Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84646538240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121631494144, + "active_bytes_peak_delta": 31186074112, + "after": { + "active_bytes_current": 84646538240, + "active_bytes_peak": 121631494144, + "allocated_bytes_current": 84646538240, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5153510400, + "inactive_split_bytes_peak": 14758083584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122838581248, + "reserved_bytes_peak": 122838581248, + "segment_count_current": 3019, + "segment_count_peak": 3019, + "timestamp_utc": "2026-08-26T18:55:53.843637Z", + "world_size": 8 + }, + "allocated_bytes_current": 84646538240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121178628096, + "allocated_bytes_peak_delta": 30733208064, + "before": { + "active_bytes_current": 84646538240, + "active_bytes_peak": 90445420032, + "allocated_bytes_current": 84646538240, + "allocated_bytes_peak": 90445420032, + "device": 3, + "inactive_split_bytes_current": 4950086656, + "inactive_split_bytes_peak": 14758083584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 91775565824, + "reserved_bytes_peak": 91775565824, + "segment_count_current": 2239, + "segment_count_peak": 2239, + "timestamp_utc": "2026-08-26T18:55:52.421280Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5153510400, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 14758083584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122838581248, + "reserved_bytes_current_delta": 31063015424, + "reserved_bytes_peak": 122838581248, + "reserved_bytes_peak_delta": 31063015424, + "segment_count_current": 3019, + "segment_count_current_delta": 780, + "segment_count_peak": 3019, + "segment_count_peak_delta": 780, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185553844080Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.035722Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185553844080Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185553844080Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35ea73a97c31b4d472efc349de1543a1f617be30 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185553844080Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fb1e3907c3be6f850896303255c3106b22a903cb69448b1954c177b60a14190 +size 10182222 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185555302847Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185555302847Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c811fa9d2217ba8b448dc6c90f1e5bf6deb3259 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185555302847Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84646538240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121631494144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84646538240, + "active_bytes_peak": 121631494144, + "allocated_bytes_current": 84646538240, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5054944256, + "inactive_split_bytes_peak": 14758083584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122838581248, + "reserved_bytes_peak": 122838581248, + "segment_count_current": 3019, + "segment_count_peak": 3019, + "timestamp_utc": "2026-08-26T18:55:55.302407Z", + "world_size": 8 + }, + "allocated_bytes_current": 84646538240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121178628096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84646538240, + "active_bytes_peak": 121631494144, + "allocated_bytes_current": 84646538240, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5153510400, + "inactive_split_bytes_peak": 14758083584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122838581248, + "reserved_bytes_peak": 122838581248, + "segment_count_current": 3019, + "segment_count_peak": 3019, + "timestamp_utc": "2026-08-26T18:55:54.061413Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5054944256, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14758083584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122838581248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122838581248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3019, + "segment_count_current_delta": 0, + "segment_count_peak": 3019, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185555302847Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.496502Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185555302847Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185555302847Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0b0e7633cf000b85f873dab0a8ed527f8ccc50e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185555302847Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000d80ba61e2ead7889a82f5a04df9ae4cd7a7d65a6d6a937331a52b11b80a10 +size 9690749 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185559304568Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185559304568Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9974354bf73d0fd51c9a34217a24799136567f6b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185559304568Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84663581696, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127846960640, + "active_bytes_peak_delta": 6215466496, + "after": { + "active_bytes_current": 84663581696, + "active_bytes_peak": 127846960640, + "allocated_bytes_current": 84663581696, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_peak": 14758083584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142354677760, + "reserved_bytes_peak": 142354677760, + "segment_count_current": 3424, + "segment_count_peak": 3424, + "timestamp_utc": "2026-08-26T18:55:59.303968Z", + "world_size": 8 + }, + "allocated_bytes_current": 84663581696, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121178628096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84646538240, + "active_bytes_peak": 121631494144, + "allocated_bytes_current": 84646538240, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5054944256, + "inactive_split_bytes_peak": 14758083584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122838581248, + "reserved_bytes_peak": 122838581248, + "segment_count_current": 3019, + "segment_count_peak": 3019, + "timestamp_utc": "2026-08-26T18:55:55.521844Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_current_delta": -48500736, + "inactive_split_bytes_peak": 14758083584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142354677760, + "reserved_bytes_current_delta": 19516096512, + "reserved_bytes_peak": 142354677760, + "reserved_bytes_peak_delta": 19516096512, + "segment_count_current": 3424, + "segment_count_current_delta": 405, + "segment_count_peak": 3424, + "segment_count_peak_delta": 405, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185559304568Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.511255Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185559304568Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185559304568Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a91282a85cfbab37b51f642709a57315ed71aa9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185559304568Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:541bb3d2dec794561f1dbd355f6ff8148423ffa3c891b43106e4998b591627b9 +size 13053805 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185559950907Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185559950907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a55a44ecb72597b5fee2c9023cb543db2aa3d6e7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185559950907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84663581696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127846960640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84663581696, + "active_bytes_peak": 127846960640, + "allocated_bytes_current": 84663581696, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_peak": 14758083584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142354677760, + "reserved_bytes_peak": 142354677760, + "segment_count_current": 3424, + "segment_count_peak": 3424, + "timestamp_utc": "2026-08-26T18:55:59.950444Z", + "world_size": 8 + }, + "allocated_bytes_current": 84663581696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121178628096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84663581696, + "active_bytes_peak": 127846960640, + "allocated_bytes_current": 84663581696, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_peak": 14758083584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142354677760, + "reserved_bytes_peak": 142354677760, + "segment_count_current": 3424, + "segment_count_peak": 3424, + "timestamp_utc": "2026-08-26T18:55:59.538219Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14758083584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142354677760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142354677760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3424, + "segment_count_current_delta": 0, + "segment_count_peak": 3424, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185559950907Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.977330Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185559950907Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185559950907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a600b2fc170d1e40d9f55bfda23fc3a04d861ad8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185559950907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7aa827cf31cb838e765b6eebd054cf74033fdd972a1cc7a318ae22644657687 +size 1153030 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185600373595Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185600373595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af5d9d9357c5a92ec998cb4a4f00d3b6c1e669fb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185600373595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84663581696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127846960640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84663581696, + "active_bytes_peak": 127846960640, + "allocated_bytes_current": 84663581696, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_peak": 14758083584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142354677760, + "reserved_bytes_peak": 142354677760, + "segment_count_current": 3424, + "segment_count_peak": 3424, + "timestamp_utc": "2026-08-26T18:56:00.373186Z", + "world_size": 8 + }, + "allocated_bytes_current": 84663581696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121178628096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84663581696, + "active_bytes_peak": 127846960640, + "allocated_bytes_current": 84663581696, + "allocated_bytes_peak": 121178628096, + "device": 3, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_peak": 14758083584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142354677760, + "reserved_bytes_peak": 142354677760, + "segment_count_current": 3424, + "segment_count_peak": 3424, + "timestamp_utc": "2026-08-26T18:55:59.979061Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5006443520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14758083584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142354677760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142354677760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3424, + "segment_count_current_delta": 0, + "segment_count_peak": 3424, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185600373595Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.396002Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185600373595Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185600373595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2927c054523516b8369ce066a2a8089df206f05 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185600373595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dc622a133b54a742d42d371af1c3f9fb8113243193770446560e67d05506da5 +size 1153030 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185552197079Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185552197079Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9fd50bb7c9c4bc3ef3da233a49ac564faeb3bbf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185552197079Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84646095360, + "active_bytes_current_delta": 41095417856, + "active_bytes_peak": 90444027904, + "active_bytes_peak_delta": 46887058944, + "after": { + "active_bytes_current": 84646095360, + "active_bytes_peak": 90444027904, + "allocated_bytes_current": 84646095360, + "allocated_bytes_peak": 90444027904, + "device": 4, + "inactive_split_bytes_current": 4684191232, + "inactive_split_bytes_peak": 14442123264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 91477770240, + "reserved_bytes_peak": 91477770240, + "segment_count_current": 2237, + "segment_count_peak": 2237, + "timestamp_utc": "2026-08-26T18:55:52.196429Z", + "world_size": 8 + }, + "allocated_bytes_current": 84646095360, + "allocated_bytes_current_delta": 41095417856, + "allocated_bytes_peak": 90444027904, + "allocated_bytes_peak_delta": 46887058944, + "before": { + "active_bytes_current": 43550677504, + "active_bytes_peak": 43556968960, + "allocated_bytes_current": 43550677504, + "allocated_bytes_peak": 43556968960, + "device": 4, + "inactive_split_bytes_current": 1252877824, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T18:55:12.944549Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4684191232, + "inactive_split_bytes_current_delta": 3431313408, + "inactive_split_bytes_peak": 14442123264, + "inactive_split_bytes_peak_delta": 12829210624, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 91477770240, + "reserved_bytes_current_delta": 46657437696, + "reserved_bytes_peak": 91477770240, + "reserved_bytes_peak_delta": 46657437696, + "segment_count_current": 2237, + "segment_count_current_delta": 1348, + "segment_count_peak": 2237, + "segment_count_peak_delta": 1348, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185552197079Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.390512Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185552197079Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185552197079Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..854372fba95bd94b4165476510a56020b591075c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185552197079Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56d88a1b4994946400e3c7076bf9d60e02e4c2a3288b396ca5738912e96f50a0 +size 9760051 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185553844326Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185553844326Z.json new file mode 100644 index 0000000000000000000000000000000000000000..009c4cf58a01694ec2d193c130d5d65f3612a3c3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185553844326Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84646095360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121226076672, + "active_bytes_peak_delta": 30782048768, + "after": { + "active_bytes_current": 84646095360, + "active_bytes_peak": 121226076672, + "allocated_bytes_current": 84646095360, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4585625088, + "inactive_split_bytes_peak": 14442123264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 122276544512, + "reserved_bytes_peak": 122276544512, + "segment_count_current": 3015, + "segment_count_peak": 3015, + "timestamp_utc": "2026-08-26T18:55:53.843805Z", + "world_size": 8 + }, + "allocated_bytes_current": 84646095360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121226076672, + "allocated_bytes_peak_delta": 30782048768, + "before": { + "active_bytes_current": 84646095360, + "active_bytes_peak": 90444027904, + "allocated_bytes_current": 84646095360, + "allocated_bytes_peak": 90444027904, + "device": 4, + "inactive_split_bytes_current": 4684191232, + "inactive_split_bytes_peak": 14442123264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 91477770240, + "reserved_bytes_peak": 91477770240, + "segment_count_current": 2237, + "segment_count_peak": 2237, + "timestamp_utc": "2026-08-26T18:55:52.426834Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4585625088, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14442123264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 122276544512, + "reserved_bytes_current_delta": 30798774272, + "reserved_bytes_peak": 122276544512, + "reserved_bytes_peak_delta": 30798774272, + "segment_count_current": 3015, + "segment_count_current_delta": 778, + "segment_count_peak": 3015, + "segment_count_peak_delta": 778, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185553844326Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.054517Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185553844326Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185553844326Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fbcaf73cfd6680a93fdaa125904f1c72f9642ef7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185553844326Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c9dbf6cd78d8872ffd86bfff53f725a2f1c94048fce1910349041a77d0b8be2 +size 10159891 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185555303125Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185555303125Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e3cdc40f1f386caba5e15b3d90f1f145dd64702 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185555303125Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84646095360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121226076672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84646095360, + "active_bytes_peak": 121226076672, + "allocated_bytes_current": 84646095360, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4686288384, + "inactive_split_bytes_peak": 14442123264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 122276544512, + "reserved_bytes_peak": 122276544512, + "segment_count_current": 3015, + "segment_count_peak": 3015, + "timestamp_utc": "2026-08-26T18:55:55.302619Z", + "world_size": 8 + }, + "allocated_bytes_current": 84646095360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121226076672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84646095360, + "active_bytes_peak": 121226076672, + "allocated_bytes_current": 84646095360, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4585625088, + "inactive_split_bytes_peak": 14442123264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 122276544512, + "reserved_bytes_peak": 122276544512, + "segment_count_current": 3015, + "segment_count_peak": 3015, + "timestamp_utc": "2026-08-26T18:55:54.083553Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4686288384, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14442123264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 122276544512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122276544512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3015, + "segment_count_current_delta": 0, + "segment_count_peak": 3015, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185555303125Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.514427Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185555303125Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185555303125Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5c63b400f2c40c0b7721611aa7b660fcffc4589 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185555303125Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee3cb2be5a4a5f811fbad02c92a2c3e9bb6dc30b4cc984e106c4d7dc99820844 +size 10136876 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185559304755Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185559304755Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42b7b8aecb8e4e786933dee60ff50e161bc00d15 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185559304755Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84663138816, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127846517760, + "active_bytes_peak_delta": 6620441088, + "after": { + "active_bytes_current": 84663138816, + "active_bytes_peak": 127846517760, + "allocated_bytes_current": 84663138816, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_peak": 14442123264, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141985579008, + "reserved_bytes_peak": 141985579008, + "segment_count_current": 3420, + "segment_count_peak": 3420, + "timestamp_utc": "2026-08-26T18:55:59.304153Z", + "world_size": 8 + }, + "allocated_bytes_current": 84663138816, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121226076672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84646095360, + "active_bytes_peak": 121226076672, + "allocated_bytes_current": 84646095360, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4686288384, + "inactive_split_bytes_peak": 14442123264, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 122276544512, + "reserved_bytes_peak": 122276544512, + "segment_count_current": 3015, + "segment_count_peak": 3015, + "timestamp_utc": "2026-08-26T18:55:55.543503Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_current_delta": 52162560, + "inactive_split_bytes_peak": 14442123264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141985579008, + "reserved_bytes_current_delta": 19709034496, + "reserved_bytes_peak": 141985579008, + "reserved_bytes_peak_delta": 19709034496, + "segment_count_current": 3420, + "segment_count_current_delta": 405, + "segment_count_peak": 3420, + "segment_count_peak_delta": 405, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185559304755Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.517806Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185559304755Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185559304755Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d501cfb27ca8fd200013fb2a43f1ad792ff15fd7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185559304755Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ed05bc23e9553b00468af30e517a6f45a63770281f923ad19c8a76838abde1f +size 13057214 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185559951106Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185559951106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f5c94f35b038e64f9bf174fd66f97fb269e2645 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185559951106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84663138816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127846517760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84663138816, + "active_bytes_peak": 127846517760, + "allocated_bytes_current": 84663138816, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_peak": 14442123264, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141985579008, + "reserved_bytes_peak": 141985579008, + "segment_count_current": 3420, + "segment_count_peak": 3420, + "timestamp_utc": "2026-08-26T18:55:59.950593Z", + "world_size": 8 + }, + "allocated_bytes_current": 84663138816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121226076672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84663138816, + "active_bytes_peak": 127846517760, + "allocated_bytes_current": 84663138816, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_peak": 14442123264, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141985579008, + "reserved_bytes_peak": 141985579008, + "segment_count_current": 3420, + "segment_count_peak": 3420, + "timestamp_utc": "2026-08-26T18:55:59.546574Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14442123264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141985579008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141985579008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3420, + "segment_count_current_delta": 0, + "segment_count_peak": 3420, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185559951106Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.979227Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185559951106Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185559951106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0e86ba557dc0b8b379c151f815ea4a9ffe609b3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185559951106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:602d115efdd80b7d9112ca2bbfc7495be399f9a3c4414f3009f9ebae5bd4c70d +size 1154243 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185600373730Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185600373730Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d5a66091c92cd4c7533deb85c400ff30d2abe7b8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185600373730Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84663138816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127846517760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84663138816, + "active_bytes_peak": 127846517760, + "allocated_bytes_current": 84663138816, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_peak": 14442123264, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141985579008, + "reserved_bytes_peak": 141985579008, + "segment_count_current": 3420, + "segment_count_peak": 3420, + "timestamp_utc": "2026-08-26T18:56:00.373319Z", + "world_size": 8 + }, + "allocated_bytes_current": 84663138816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121226076672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84663138816, + "active_bytes_peak": 127846517760, + "allocated_bytes_current": 84663138816, + "allocated_bytes_peak": 121226076672, + "device": 4, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_peak": 14442123264, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141985579008, + "reserved_bytes_peak": 141985579008, + "segment_count_current": 3420, + "segment_count_peak": 3420, + "timestamp_utc": "2026-08-26T18:55:59.981025Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4738450944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14442123264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141985579008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141985579008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3420, + "segment_count_current_delta": 0, + "segment_count_peak": 3420, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185600373730Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.396877Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185600373730Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185600373730Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..277ca3a42ba77ecbce30b07c9c5d443e05d2c806 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185600373730Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4188acef3c748900d9f95d94fd1359950804c9a3f43e327087039bcff9c39238 +size 1154243 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185552196687Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185552196687Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d386fb735eb669d0e701eea71765f8a1014b3cf5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185552196687Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 41083820544, + "active_bytes_peak": 90446782464, + "active_bytes_peak_delta": 46886947840, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 90446782464, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 90446782464, + "device": 5, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_peak": 14460597760, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 91565850624, + "reserved_bytes_peak": 91565850624, + "segment_count_current": 2224, + "segment_count_peak": 2224, + "timestamp_utc": "2026-08-26T18:55:52.196237Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 41083820544, + "allocated_bytes_peak": 90446782464, + "allocated_bytes_peak_delta": 46886947840, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 5, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:55:12.944314Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_current_delta": 3400967680, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 12847685120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 91565850624, + "reserved_bytes_current_delta": 46730838016, + "reserved_bytes_peak": 91565850624, + "reserved_bytes_peak_delta": 46730838016, + "segment_count_current": 2224, + "segment_count_current_delta": 1341, + "segment_count_peak": 2224, + "segment_count_peak_delta": 1341, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185552196687Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.390466Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185552196687Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185552196687Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df2904233852909565e6b6e2662aeef5fd833188 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185552196687Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39a3011ffd6ace6bd5114ad5c798d7333d95b0a388b688e29a0ba4f8b2c0cd2d +size 9740495 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185553844096Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185553844096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e462135f2283dfe5e0a46bb3841db52cbfaa6422 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185553844096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121666840064, + "active_bytes_peak_delta": 31220057600, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_peak": 14460597760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:53.843664Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 30767072768, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 90446782464, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 90446782464, + "device": 5, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_peak": 14460597760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 91565850624, + "reserved_bytes_peak": 91565850624, + "segment_count_current": 2224, + "segment_count_peak": 2224, + "timestamp_utc": "2026-08-26T18:55:52.424854Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 122853261312, + "reserved_bytes_current_delta": 31287410688, + "reserved_bytes_peak": 122853261312, + "reserved_bytes_peak_delta": 31287410688, + "segment_count_current": 2996, + "segment_count_current_delta": 772, + "segment_count_peak": 2996, + "segment_count_peak_delta": 772, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185553844096Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.036888Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185553844096Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185553844096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a616d61914d7502a5a4522e87a235f8bfb728e9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185553844096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1488abefa53c0d129ce2641c472a41a5c2ce6dd6c106346c51f22caffb8dbc1a +size 10167803 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185555302887Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185555302887Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4cce5d24c84c3d6955a8093817fab9d8809b2307 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185555302887Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121666840064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_peak": 14460597760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:55.302457Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_peak": 14460597760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:54.063517Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 122853261312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122853261312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2996, + "segment_count_current_delta": 0, + "segment_count_peak": 2996, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185555302887Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.494839Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185555302887Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185555302887Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c3557a387c52414e1f9cd3064a3ae5b90c9a3ba --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185555302887Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd5a2c73e088d12652c1617e5c407ba68cc0a3b5d6233511e829aea2c2d0509f +size 9669790 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185559304512Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185559304512Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bfc0f9cc19a413870c8f38612dbd00a3ac01ef16 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185559304512Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 6170946048, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142153351168, + "reserved_bytes_peak": 142153351168, + "segment_count_current": 3398, + "segment_count_peak": 3398, + "timestamp_utc": "2026-08-26T18:55:59.303982Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_peak": 14460597760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:55.520286Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": -151261184, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142153351168, + "reserved_bytes_current_delta": 19300089856, + "reserved_bytes_peak": 142153351168, + "reserved_bytes_peak_delta": 19300089856, + "segment_count_current": 3398, + "segment_count_current_delta": 402, + "segment_count_peak": 3398, + "segment_count_peak_delta": 402, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185559304512Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.508271Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185559304512Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185559304512Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aefe880c72b2642028f911800c89c1252e73a54a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185559304512Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6852b44b8356915024068dc959b233f1b87f6a5d09dc71c91a3522be38772ebd +size 11671777 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185559950938Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185559950938Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19066748fcf4feded5fb6d15c66e3ec67affd300 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185559950938Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142153351168, + "reserved_bytes_peak": 142153351168, + "segment_count_current": 3398, + "segment_count_peak": 3398, + "timestamp_utc": "2026-08-26T18:55:59.950469Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142153351168, + "reserved_bytes_peak": 142153351168, + "segment_count_current": 3398, + "segment_count_peak": 3398, + "timestamp_utc": "2026-08-26T18:55:59.535723Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142153351168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142153351168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3398, + "segment_count_current_delta": 0, + "segment_count_peak": 3398, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185559950938Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.976798Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185559950938Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185559950938Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..952f9f761d6a1b735dfd4d3d708e2d5d78a281a7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185559950938Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a7a56f862f0cc447c66accbc860cfc78c6849336a338fe6613c8b20fcfda289 +size 1030869 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185600373595Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185600373595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..62ca2fe1128a38eab4175d0d7cbddc9721ab3ba6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185600373595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142153351168, + "reserved_bytes_peak": 142153351168, + "segment_count_current": 3398, + "segment_count_peak": 3398, + "timestamp_utc": "2026-08-26T18:56:00.373195Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 5, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142153351168, + "reserved_bytes_peak": 142153351168, + "segment_count_current": 3398, + "segment_count_peak": 3398, + "timestamp_utc": "2026-08-26T18:55:59.978519Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142153351168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142153351168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3398, + "segment_count_current_delta": 0, + "segment_count_peak": 3398, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185600373595Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.395557Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185600373595Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185600373595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9fc587d7ccb07bc647e90c82364e5fcfc47af42e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185600373595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f91c4a4b9e20a36d143e434e75c17fc0216312d0bc882d9127e1d3d99e750292 +size 1030869 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185552196986Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185552196986Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5edc7a1719525fc0cf3fb506cde623bde2b48fd8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185552196986Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84633959424, + "active_bytes_current_delta": 41080548864, + "active_bytes_peak": 90440307712, + "active_bytes_peak_delta": 46880605696, + "after": { + "active_bytes_current": 84633959424, + "active_bytes_peak": 90440307712, + "allocated_bytes_current": 84633959424, + "allocated_bytes_peak": 90440307712, + "device": 6, + "inactive_split_bytes_current": 4950082560, + "inactive_split_bytes_peak": 14802503680, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 91360329728, + "reserved_bytes_peak": 91360329728, + "segment_count_current": 2236, + "segment_count_peak": 2236, + "timestamp_utc": "2026-08-26T18:55:52.196397Z", + "world_size": 8 + }, + "allocated_bytes_current": 84633959424, + "allocated_bytes_current_delta": 41080548864, + "allocated_bytes_peak": 90440307712, + "allocated_bytes_peak_delta": 46880605696, + "before": { + "active_bytes_current": 43553410560, + "active_bytes_peak": 43559702016, + "allocated_bytes_current": 43553410560, + "allocated_bytes_peak": 43559702016, + "device": 6, + "inactive_split_bytes_current": 1256436224, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:55:12.944594Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4950082560, + "inactive_split_bytes_current_delta": 3693646336, + "inactive_split_bytes_peak": 14802503680, + "inactive_split_bytes_peak_delta": 13189591040, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 91360329728, + "reserved_bytes_current_delta": 46535802880, + "reserved_bytes_peak": 91360329728, + "reserved_bytes_peak_delta": 46535802880, + "segment_count_current": 2236, + "segment_count_current_delta": 1346, + "segment_count_peak": 2236, + "segment_count_peak_delta": 1346, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185552196986Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.387321Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185552196986Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185552196986Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a3029f970388897ff6411405fbc1185c0380813 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185552196986Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dd613748923070e737c04d3716442a4cef571e102a93135ed4686846ae5e5ef +size 9760120 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185553844179Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185553844179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ddb7582789995857460684886a62738e7c128d29 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185553844179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84633959424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121325399552, + "active_bytes_peak_delta": 30885091840, + "after": { + "active_bytes_current": 84633959424, + "active_bytes_peak": 121325399552, + "allocated_bytes_current": 84633959424, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5052843008, + "inactive_split_bytes_peak": 14802503680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 122524008448, + "reserved_bytes_peak": 122524008448, + "segment_count_current": 3017, + "segment_count_peak": 3017, + "timestamp_utc": "2026-08-26T18:55:53.843748Z", + "world_size": 8 + }, + "allocated_bytes_current": 84633959424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121174404608, + "allocated_bytes_peak_delta": 30734096896, + "before": { + "active_bytes_current": 84633959424, + "active_bytes_peak": 90440307712, + "allocated_bytes_current": 84633959424, + "allocated_bytes_peak": 90440307712, + "device": 6, + "inactive_split_bytes_current": 4950082560, + "inactive_split_bytes_peak": 14802503680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 91360329728, + "reserved_bytes_peak": 91360329728, + "segment_count_current": 2236, + "segment_count_peak": 2236, + "timestamp_utc": "2026-08-26T18:55:52.419353Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5052843008, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 14802503680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 122524008448, + "reserved_bytes_current_delta": 31163678720, + "reserved_bytes_peak": 122524008448, + "reserved_bytes_peak_delta": 31163678720, + "segment_count_current": 3017, + "segment_count_current_delta": 781, + "segment_count_peak": 3017, + "segment_count_peak_delta": 781, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185553844179Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.035488Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185553844179Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185553844179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0161c7b07d3590280756973cf77b7b95569192c4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185553844179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d24a2299072c80455881b5e0c7021b53fa00ee016e38740c28d32ec38235036 +size 10183602 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185555303007Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185555303007Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44143c1bfa364477229181d9309531995b6d6c3a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185555303007Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84633959424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121620606464, + "active_bytes_peak_delta": 295206912, + "after": { + "active_bytes_current": 84633959424, + "active_bytes_peak": 121620606464, + "allocated_bytes_current": 84633959424, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5155603456, + "inactive_split_bytes_peak": 14802503680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 122825998336, + "reserved_bytes_peak": 122825998336, + "segment_count_current": 3020, + "segment_count_peak": 3020, + "timestamp_utc": "2026-08-26T18:55:55.302577Z", + "world_size": 8 + }, + "allocated_bytes_current": 84633959424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121174404608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84633959424, + "active_bytes_peak": 121325399552, + "allocated_bytes_current": 84633959424, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5052843008, + "inactive_split_bytes_peak": 14802503680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 122524008448, + "reserved_bytes_peak": 122524008448, + "segment_count_current": 3017, + "segment_count_peak": 3017, + "timestamp_utc": "2026-08-26T18:55:54.061750Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5155603456, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 14802503680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 122825998336, + "reserved_bytes_current_delta": 301989888, + "reserved_bytes_peak": 122825998336, + "reserved_bytes_peak_delta": 301989888, + "segment_count_current": 3020, + "segment_count_current_delta": 3, + "segment_count_peak": 3020, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185555303007Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.494838Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185555303007Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185555303007Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4554f3f443b1eca720d442d8f61fbb500a32514 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185555303007Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a409eb2a233c3005bdbe6c79b4d05a543909e7e83f8b4e0d42433af83c2e901e +size 9692825 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185559304868Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185559304868Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd500792eee0d08cc56328fb69cd86b1f9378d3d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185559304868Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651002880, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127834381824, + "active_bytes_peak_delta": 6213775360, + "after": { + "active_bytes_current": 84651002880, + "active_bytes_peak": 127834381824, + "allocated_bytes_current": 84651002880, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_peak": 14802503680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142442758144, + "reserved_bytes_peak": 142442758144, + "segment_count_current": 3426, + "segment_count_peak": 3426, + "timestamp_utc": "2026-08-26T18:55:59.304278Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651002880, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121174404608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84633959424, + "active_bytes_peak": 121620606464, + "allocated_bytes_current": 84633959424, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5155603456, + "inactive_split_bytes_peak": 14802503680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 122825998336, + "reserved_bytes_peak": 122825998336, + "segment_count_current": 3020, + "segment_count_peak": 3020, + "timestamp_utc": "2026-08-26T18:55:55.521132Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_current_delta": -149164032, + "inactive_split_bytes_peak": 14802503680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142442758144, + "reserved_bytes_current_delta": 19616759808, + "reserved_bytes_peak": 142442758144, + "reserved_bytes_peak_delta": 19616759808, + "segment_count_current": 3426, + "segment_count_current_delta": 406, + "segment_count_peak": 3426, + "segment_count_peak_delta": 406, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185559304868Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.516242Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185559304868Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185559304868Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b0804381cbcb87f4cf16b71f2d63aee585ed3558 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185559304868Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24759a14e9336ea9a235811fdb83619fa95438824fec652f54520f4c86ccd7c3 +size 13055968 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185559951099Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185559951099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b88563ca1fe6f66a6962be59e2f94c7ced8214c9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185559951099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651002880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127834381824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84651002880, + "active_bytes_peak": 127834381824, + "allocated_bytes_current": 84651002880, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_peak": 14802503680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142442758144, + "reserved_bytes_peak": 142442758144, + "segment_count_current": 3426, + "segment_count_peak": 3426, + "timestamp_utc": "2026-08-26T18:55:59.950583Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651002880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121174404608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84651002880, + "active_bytes_peak": 127834381824, + "allocated_bytes_current": 84651002880, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_peak": 14802503680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142442758144, + "reserved_bytes_peak": 142442758144, + "segment_count_current": 3426, + "segment_count_peak": 3426, + "timestamp_utc": "2026-08-26T18:55:59.548485Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14802503680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142442758144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142442758144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3426, + "segment_count_current_delta": 0, + "segment_count_peak": 3426, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185559951099Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.978220Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185559951099Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185559951099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69b0b7abef0514303d40a10fe2eb7cd2b05e1944 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185559951099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b1bb2c9e0dbbba6465ab2b84864baceaf41a63aa4f1d587531b5c11a5c8e776 +size 1155284 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185600373717Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185600373717Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ba2dd32732db27e098b04235d1c950e131a517b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185600373717Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651002880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127834381824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84651002880, + "active_bytes_peak": 127834381824, + "allocated_bytes_current": 84651002880, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_peak": 14802503680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142442758144, + "reserved_bytes_peak": 142442758144, + "segment_count_current": 3426, + "segment_count_peak": 3426, + "timestamp_utc": "2026-08-26T18:56:00.373303Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651002880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121174404608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84651002880, + "active_bytes_peak": 127834381824, + "allocated_bytes_current": 84651002880, + "allocated_bytes_peak": 121174404608, + "device": 6, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_peak": 14802503680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142442758144, + "reserved_bytes_peak": 142442758144, + "segment_count_current": 3426, + "segment_count_peak": 3426, + "timestamp_utc": "2026-08-26T18:55:59.980052Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5006439424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14802503680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142442758144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142442758144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3426, + "segment_count_current_delta": 0, + "segment_count_peak": 3426, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185600373717Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.396783Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185600373717Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185600373717Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3e6540b10ba11b8c8f2c405e7bc50a0ddea3a31 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185600373717Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7793951d497a8eb070c73cdbe97dcdff55ae32810e0e13beb867b60f1bc4926 +size 1155284 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185552196674Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185552196674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..295b464b9b61b4f567dbe39ddb35adae69757ecd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185552196674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 41083820544, + "active_bytes_peak": 90446782464, + "active_bytes_peak_delta": 46886947840, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 90446782464, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 90446782464, + "device": 7, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_peak": 14460597760, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 91465187328, + "reserved_bytes_peak": 91465187328, + "segment_count_current": 2223, + "segment_count_peak": 2223, + "timestamp_utc": "2026-08-26T18:55:52.196140Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 41083820544, + "allocated_bytes_peak": 90446782464, + "allocated_bytes_peak_delta": 46886947840, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 7, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:55:12.944121Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_current_delta": 3400967680, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 12847685120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 91465187328, + "reserved_bytes_current_delta": 46630174720, + "reserved_bytes_peak": 91465187328, + "reserved_bytes_peak_delta": 46630174720, + "segment_count_current": 2223, + "segment_count_current_delta": 1340, + "segment_count_peak": 2223, + "segment_count_peak_delta": 1340, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185552196674Z.pickle", + "timestamp_utc": "2026-08-26T18:55:52.391175Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185552196674Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185552196674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..997905595c970b73e96774f35662ce28986523b7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185552196674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd7d9e7d472322375f1aa05ef7183e72e9b3aab622b9d3b45cddf8c8a6f4040e +size 9739333 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185553844119Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185553844119Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0eb499821822012b29f765f9cc8b02c6e550b88c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185553844119Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121666840064, + "active_bytes_peak_delta": 31220057600, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_peak": 14460597760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:53.843682Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 30767072768, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 90446782464, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 90446782464, + "device": 7, + "inactive_split_bytes_current": 4657271296, + "inactive_split_bytes_peak": 14460597760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 91465187328, + "reserved_bytes_peak": 91465187328, + "segment_count_current": 2223, + "segment_count_peak": 2223, + "timestamp_utc": "2026-08-26T18:55:52.430533Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122853261312, + "reserved_bytes_current_delta": 31388073984, + "reserved_bytes_peak": 122853261312, + "reserved_bytes_peak_delta": 31388073984, + "segment_count_current": 2996, + "segment_count_current_delta": 773, + "segment_count_peak": 2996, + "segment_count_peak_delta": 773, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185553844119Z.pickle", + "timestamp_utc": "2026-08-26T18:55:54.036722Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185553844119Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185553844119Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6afb8250b4ee60f9a237e16a9bd4d5a1b890effb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185553844119Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:595ab070d80a8ba31e32dc0a403e8ec5f02b3b88b23134d9eb50a98cc19d2b77 +size 10168047 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185555302828Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185555302828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56b12a77b46a38fceddd8a65ffe500b7e3fa6e23 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185555302828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637363712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121666840064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4661465600, + "inactive_split_bytes_peak": 14460597760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:55.302389Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637363712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4862792192, + "inactive_split_bytes_peak": 14460597760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:54.063236Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4661465600, + "inactive_split_bytes_current_delta": -201326592, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122853261312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122853261312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2996, + "segment_count_current_delta": 0, + "segment_count_peak": 2996, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185555302828Z.pickle", + "timestamp_utc": "2026-08-26T18:55:55.500436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185555302828Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185555302828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bcf1e8532721768fc462a6b08b79cbc1c25d4a98 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185555302828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90d8ee672c9bfd01dda3b66694710d4b367eb74b59ec6cff8a0294c11078d568 +size 9676677 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185559304590Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185559304590Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8903d4ed43e412579f26e4c49ab377c9e45b3dbe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185559304590Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 6170946048, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 141952024576, + "reserved_bytes_peak": 141952024576, + "segment_count_current": 3396, + "segment_count_peak": 3396, + "timestamp_utc": "2026-08-26T18:55:59.304024Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637363712, + "active_bytes_peak": 121666840064, + "allocated_bytes_current": 84637363712, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4661465600, + "inactive_split_bytes_peak": 14460597760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122853261312, + "reserved_bytes_peak": 122853261312, + "segment_count_current": 2996, + "segment_count_peak": 2996, + "timestamp_utc": "2026-08-26T18:55:55.526393Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": 50065408, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 141952024576, + "reserved_bytes_current_delta": 19098763264, + "reserved_bytes_peak": 141952024576, + "reserved_bytes_peak_delta": 19098763264, + "segment_count_current": 3396, + "segment_count_current_delta": 400, + "segment_count_peak": 3396, + "segment_count_peak_delta": 400, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185559304590Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.517457Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185559304590Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185559304590Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..110e8d05e35005108769f9310442119c0de95447 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185559304590Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb6d89040314ad4320e879a5ee2a1b37e7d793b2db1b60891cb590ff2dd803eb +size 11671897 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185559950898Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185559950898Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50b21c1210b02ff3c9e9b288de6f7b656f363011 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185559950898Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 141952024576, + "reserved_bytes_peak": 141952024576, + "segment_count_current": 3396, + "segment_count_peak": 3396, + "timestamp_utc": "2026-08-26T18:55:59.950430Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 141952024576, + "reserved_bytes_peak": 141952024576, + "segment_count_current": 3396, + "segment_count_peak": 3396, + "timestamp_utc": "2026-08-26T18:55:59.545483Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 141952024576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141952024576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3396, + "segment_count_current_delta": 0, + "segment_count_peak": 3396, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185559950898Z.pickle", + "timestamp_utc": "2026-08-26T18:55:59.976244Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185559950898Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185559950898Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f689290af9df528463ee434a5b7ea00e656b693a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185559950898Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1543a01fe94bbf2a6582f4bbbfef00326a3f04f9cfb340978af3026ccfa5da52 +size 1031089 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185600373620Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185600373620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68d8d8d1a6a0da26969d378ee057eec805e6bb4d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185600373620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84654407168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127837786112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 141952024576, + "reserved_bytes_peak": 141952024576, + "segment_count_current": 3396, + "segment_count_peak": 3396, + "timestamp_utc": "2026-08-26T18:56:00.373196Z", + "world_size": 8 + }, + "allocated_bytes_current": 84654407168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121213855232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84654407168, + "active_bytes_peak": 127837786112, + "allocated_bytes_current": 84654407168, + "allocated_bytes_peak": 121213855232, + "device": 7, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_peak": 14460597760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 141952024576, + "reserved_bytes_peak": 141952024576, + "segment_count_current": 3396, + "segment_count_peak": 3396, + "timestamp_utc": "2026-08-26T18:55:59.978020Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4711531008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14460597760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 141952024576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141952024576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3396, + "segment_count_current_delta": 0, + "segment_count_peak": 3396, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185600373620Z.pickle", + "timestamp_utc": "2026-08-26T18:56:00.395849Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185600373620Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185600373620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4fb593a013dacd9727da31cceb1ae7e73d4827f6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185600373620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe4ba6f93a5de05a741629c28ef07879bc4c4cb02fb0c50131816d51b71ecd1b +size 1031089 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..9bdc7b8ac65916b1370879b4b2f0ec4d393a3513 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..56c507aa31fb775062b1ed8c2e66a5cef2ae7771 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -0,0 +1,2835 @@ +[18:54:45] WARN: better-memory-dump-support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.00 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 18:54:47.083000 578167 torch/distributed/run.py:766] +W0826 18:54:47.083000 578167 torch/distributed/run.py:766] ***************************************** +W0826 18:54:47.083000 578167 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:54:47.083000 578167 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 23456 is connected to 7 is connected to is connected to is connected to is connected to 7 is connected to 7777 peer ranks. 7 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : +77 +777 + + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 18:55:05.370152731 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:55:05.370221610 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:55:05.373920111 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:55:05.373950980 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:55:05.375527245 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:55:05.375581065 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:55:05.379312924 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 18:55:05.523905725 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.974 seconds +time to initialize megatron (seconds): 7.366 +[after megatron is initialized] datetime: 2026-08-26 18:55:11 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.self_attention.linear_proj.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.self_attention.linear_qkv.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.q_layernorm.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:55:12 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 18:55:12 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (1149.38, 1309.75) + train/valid/test-data-iterators-setup ..........: (119.77, 135.57) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 18:55:12 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 18:55:52] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 39253.5 | throughput per GPU (TFLOP/s/GPU): 2.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095280E+01 | loss scale: 1.0 | grad norm: 54.348 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 80700.490234375 | max allocated: 86253.67529296875 | reserved: 87510.0 | max reserved: 87510.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 18:55:53] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 1647.7 | throughput per GPU (TFLOP/s/GPU): 54.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095501E+01 | loss scale: 1.0 | grad norm: 56.511 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:55:55] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 1458.8 | throughput per GPU (TFLOP/s/GPU): 61.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008517E+01 | loss scale: 1.0 | grad norm: 179.584 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 18:55:59] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 4001.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604467E+00 | loss scale: 1.0 | grad norm: 8.803 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:55:59] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 646.7 | throughput per GPU (TFLOP/s/GPU): 137.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.309772E+00 | loss scale: 1.0 | grad norm: 6.538 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:56:00] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 422.7 | throughput per GPU (TFLOP/s/GPU): 210.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.156965E+00 | loss scale: 1.0 | grad norm: 3.269 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:56:00 +[rank0]:[W826 18:56:01.824512931 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:56:05.769000 578167 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:56:05.770000 578167 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 578259 closing signal SIGTERM +W0826 18:56:05.770000 578167 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 578260 closing signal SIGTERM +W0826 18:56:05.771000 578167 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 578263 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 578167 got signal: 15 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..91d153f58574c9902ea116df5bbca3c21d880f82 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 47 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 38993 +1, 38993 +2, 38993 +3, 38993 +4, 38993 +5, 38993 +6, 38993 +7, 38993 +0, 44907 +1, 44859 +2, 44919 +3, 44877 +4, 44859 +5, 44897 +6, 44879 +7, 44857 +0, 45419 +1, 45473 +2, 45449 +3, 45403 +4, 45431 +5, 45427 +6, 45447 +7, 45433 +0, 46919 +1, 46921 +2, 46931 +3, 46893 +4, 46917 +5, 46931 +6, 46921 +7, 46931 +0, 47015 +1, 47017 +2, 47027 +3, 46989 +4, 47013 +5, 47027 +6, 47017 +7, 47027 +0, 47169 +1, 47181 +2, 47187 +3, 47147 +4, 47181 +5, 47201 +6, 47175 +7, 47195 +0, 47879 +1, 47881 +2, 47891 +3, 47853 +4, 47877 +5, 47891 +6, 47881 +7, 47891 +0, 48023 +1, 48025 +2, 48035 +3, 47997 +4, 48021 +5, 48035 +6, 48025 +7, 48035 +0, 48471 +1, 48473 +2, 48487 +3, 48445 +4, 48469 +5, 48487 +6, 48433 +7, 48487 +0, 48675 +1, 48677 +2, 48687 +3, 48649 +4, 48673 +5, 48687 +6, 48637 +7, 48687 +0, 92663 +1, 92671 +2, 92657 +3, 92669 +4, 92671 +5, 92663 +6, 92649 +7, 92673 +0, 93217 +1, 93243 +2, 93223 +3, 93235 +4, 93235 +5, 93223 +6, 93223 +7, 93223 +0, 93217 +1, 93243 +2, 93223 +3, 93235 +4, 93235 +5, 93223 +6, 93223 +7, 93223 +0, 93217 +1, 93243 +2, 93223 +3, 93235 +4, 93235 +5, 93223 +6, 93223 +7, 93223 +0, 93217 +1, 93243 +2, 93223 +3, 93235 +4, 93235 +5, 93223 +6, 93223 +7, 93223 +0, 94699 +1, 93957 +2, 94705 +3, 94717 +4, 94717 +5, 94705 +6, 94705 +7, 93937 +0, 95347 +1, 94701 +2, 95353 +3, 95365 +4, 95365 +5, 95353 +6, 95353 +7, 94585 +0, 95995 +1, 95393 +2, 96017 +3, 96057 +4, 95993 +5, 95997 +6, 96007 +7, 95229 +0, 125515 +1, 124835 +2, 125501 +3, 125489 +4, 125519 +5, 125501 +6, 125477 +7, 125021 +0, 93039 +1, 91863 +2, 93431 +3, 91757 +4, 91845 +5, 93725 +6, 93285 +7, 101131 +0, 143427 +1, 143359 +2, 143583 +3, 143207 +4, 143351 +5, 143679 +6, 143435 +7, 143583 +0, 143427 +1, 143359 +2, 143583 +3, 143207 +4, 143351 +5, 143679 +6, 143435 +7, 143583 +0, 142611 +1, 142543 +2, 142767 +3, 142391 +4, 2442 +5, 142863 +6, 2186 +7, 1162 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185714716391Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185714716391Z.json new file mode 100644 index 0000000000000000000000000000000000000000..439d4f4acbfa838221eb35b84e200089a99323dc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185714716391Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620611584, + "active_bytes_current_delta": 41069009408, + "active_bytes_peak": 91613433344, + "active_bytes_peak_delta": 48055539712, + "after": { + "active_bytes_current": 84620611584, + "active_bytes_peak": 91613433344, + "allocated_bytes_current": 84620611584, + "allocated_bytes_peak": 91613433344, + "device": 0, + "inactive_split_bytes_current": 4153929728, + "inactive_split_bytes_peak": 12116173824, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93289709568, + "reserved_bytes_peak": 93289709568, + "segment_count_current": 2194, + "segment_count_peak": 2194, + "timestamp_utc": "2026-08-26T18:57:14.715911Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620611584, + "allocated_bytes_current_delta": 41069009408, + "allocated_bytes_peak": 91613433344, + "allocated_bytes_peak_delta": 48055539712, + "before": { + "active_bytes_current": 43551602176, + "active_bytes_peak": 43557893632, + "allocated_bytes_current": 43551602176, + "allocated_bytes_peak": 43557893632, + "device": 0, + "inactive_split_bytes_current": 1256147456, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T18:56:36.124265Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4153929728, + "inactive_split_bytes_current_delta": 2897782272, + "inactive_split_bytes_peak": 12116173824, + "inactive_split_bytes_peak_delta": 10503261184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93289709568, + "reserved_bytes_current_delta": 48467279872, + "reserved_bytes_peak": 93289709568, + "reserved_bytes_peak_delta": 48467279872, + "segment_count_current": 2194, + "segment_count_current_delta": 1309, + "segment_count_peak": 2194, + "segment_count_peak_delta": 1309, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185714716391Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.905490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185714716391Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185714716391Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12f8aae6e6aab79ac90756505650cdb205ce5c7c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185714716391Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:481cc62b9c5e72ac6a291c5f3d87ce96605f7317dfbdcf46f697fe0b9c8e5c76 +size 9789529 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185716268547Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185716268547Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d38c97b10627e4307e7505e35268a7e851d5e6f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185716268547Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620611584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122397908480, + "active_bytes_peak_delta": 30784475136, + "after": { + "active_bytes_current": 84620611584, + "active_bytes_peak": 122397908480, + "allocated_bytes_current": 84620611584, + "allocated_bytes_peak": 122397908480, + "device": 0, + "inactive_split_bytes_current": 4055363584, + "inactive_split_bytes_peak": 12116173824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 124107358208, + "reserved_bytes_peak": 124107358208, + "segment_count_current": 2913, + "segment_count_peak": 2913, + "timestamp_utc": "2026-08-26T18:57:16.268114Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620611584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122397908480, + "allocated_bytes_peak_delta": 30784475136, + "before": { + "active_bytes_current": 84620611584, + "active_bytes_peak": 91613433344, + "allocated_bytes_current": 84620611584, + "allocated_bytes_peak": 91613433344, + "device": 0, + "inactive_split_bytes_current": 4153929728, + "inactive_split_bytes_peak": 12116173824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93289709568, + "reserved_bytes_peak": 93289709568, + "segment_count_current": 2194, + "segment_count_peak": 2194, + "timestamp_utc": "2026-08-26T18:57:14.939742Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4055363584, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12116173824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 124107358208, + "reserved_bytes_current_delta": 30817648640, + "reserved_bytes_peak": 124107358208, + "reserved_bytes_peak_delta": 30817648640, + "segment_count_current": 2913, + "segment_count_current_delta": 719, + "segment_count_peak": 2913, + "segment_count_peak_delta": 719, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185716268547Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.454132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185716268547Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185716268547Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3bab4dc475d6c23b361ce43ce82e4ff13cf97280 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185716268547Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:390d0bc6ea0f902c43b5e7a13d8dca5323e24f6ad63ded877bda309fcd7cef7c +size 10180877 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185717637260Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185717637260Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bf8cadfbe83ae3419145ff3ee5dc388c8b845be --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185717637260Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620611584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122704019456, + "active_bytes_peak_delta": 306110976, + "after": { + "active_bytes_current": 84620611584, + "active_bytes_peak": 122704019456, + "allocated_bytes_current": 84620611584, + "allocated_bytes_peak": 122402029568, + "device": 0, + "inactive_split_bytes_current": 4256690176, + "inactive_split_bytes_peak": 12116173824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 124107358208, + "reserved_bytes_peak": 124107358208, + "segment_count_current": 2913, + "segment_count_peak": 2913, + "timestamp_utc": "2026-08-26T18:57:17.636813Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620611584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122402029568, + "allocated_bytes_peak_delta": 4121088, + "before": { + "active_bytes_current": 84620611584, + "active_bytes_peak": 122397908480, + "allocated_bytes_current": 84620611584, + "allocated_bytes_peak": 122397908480, + "device": 0, + "inactive_split_bytes_current": 4055363584, + "inactive_split_bytes_peak": 12116173824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 124107358208, + "reserved_bytes_peak": 124107358208, + "segment_count_current": 2913, + "segment_count_peak": 2913, + "timestamp_utc": "2026-08-26T18:57:16.478938Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4256690176, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 12116173824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 124107358208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 124107358208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2913, + "segment_count_current_delta": 0, + "segment_count_peak": 2913, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185717637260Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.824740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185717637260Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185717637260Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e8840a22efcbfb74e66e2a52df16250cc22e802 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185717637260Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d302e9bd32833c644c2d71b545392fde3fbb1ed17f7a17dc854f83116e5ce310 +size 10160016 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185721322061Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185721322061Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ebf3e079a3ac02fc43536a550ab5dd574d869d8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185721322061Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637655040, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129014313472, + "active_bytes_peak_delta": 6310294016, + "after": { + "active_bytes_current": 84637655040, + "active_bytes_peak": 129014313472, + "allocated_bytes_current": 84637655040, + "allocated_bytes_peak": 122402029568, + "device": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_peak": 13154206720, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_peak": 142780399616, + "segment_count_current": 3339, + "segment_count_peak": 3339, + "timestamp_utc": "2026-08-26T18:57:21.321517Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637655040, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122402029568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84620611584, + "active_bytes_peak": 122704019456, + "allocated_bytes_current": 84620611584, + "allocated_bytes_peak": 122402029568, + "device": 0, + "inactive_split_bytes_current": 4256690176, + "inactive_split_bytes_peak": 12116173824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 124107358208, + "reserved_bytes_peak": 124107358208, + "segment_count_current": 2913, + "segment_count_peak": 2913, + "timestamp_utc": "2026-08-26T18:57:17.849779Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_current_delta": -65277952, + "inactive_split_bytes_peak": 13154206720, + "inactive_split_bytes_peak_delta": 1038032896, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_current_delta": 18673041408, + "reserved_bytes_peak": 142780399616, + "reserved_bytes_peak_delta": 18673041408, + "segment_count_current": 3339, + "segment_count_current_delta": 426, + "segment_count_peak": 3339, + "segment_count_peak_delta": 426, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185721322061Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.524681Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185721322061Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185721322061Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5fa8e84aa53377989c70b595c4cd981121d8712 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185721322061Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:733ae274ae73cfe7cb8a2556a002b97ac86a6c49c0d154b862c2251302782004 +size 11672734 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185721967090Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185721967090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad6cc4c3dc2031e7ca350f967401b3598a6394af --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185721967090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637655040, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129014313472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637655040, + "active_bytes_peak": 129014313472, + "allocated_bytes_current": 84637655040, + "allocated_bytes_peak": 122402029568, + "device": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_peak": 13154206720, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_peak": 142780399616, + "segment_count_current": 3339, + "segment_count_peak": 3339, + "timestamp_utc": "2026-08-26T18:57:21.966649Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637655040, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122402029568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637655040, + "active_bytes_peak": 129014313472, + "allocated_bytes_current": 84637655040, + "allocated_bytes_peak": 122402029568, + "device": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_peak": 13154206720, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_peak": 142780399616, + "segment_count_current": 3339, + "segment_count_peak": 3339, + "timestamp_utc": "2026-08-26T18:57:21.551115Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13154206720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142780399616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3339, + "segment_count_current_delta": 0, + "segment_count_peak": 3339, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185721967090Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.991189Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185721967090Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185721967090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..632410b22a615b1587acb45b9427f8e259587140 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185721967090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17fe15dce2271d9e4865262400d08f490658905add03b07e9ee085fcfdc1697a +size 1036318 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185722392831Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185722392831Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6d3e3d758caf41db09c929c8c20ff31ab5d45e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185722392831Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637655040, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129014313472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637655040, + "active_bytes_peak": 129014313472, + "allocated_bytes_current": 84637655040, + "allocated_bytes_peak": 122402029568, + "device": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_peak": 13154206720, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_peak": 142780399616, + "segment_count_current": 3339, + "segment_count_peak": 3339, + "timestamp_utc": "2026-08-26T18:57:22.392436Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637655040, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122402029568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637655040, + "active_bytes_peak": 129014313472, + "allocated_bytes_current": 84637655040, + "allocated_bytes_peak": 122402029568, + "device": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_peak": 13154206720, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_peak": 142780399616, + "segment_count_current": 3339, + "segment_count_peak": 3339, + "timestamp_utc": "2026-08-26T18:57:21.992952Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4191412224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13154206720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142780399616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142780399616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3339, + "segment_count_current_delta": 0, + "segment_count_peak": 3339, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185722392831Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.414876Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185722392831Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185722392831Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f91cbf1d4680b5f89707e7251d14fbe4fd4c439 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185722392831Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51e04da77cae68036de5e5936166a6affe8c91f11d6b59c0af317418c96a33b4 +size 1036318 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185714716389Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185714716389Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19107ad3f0f5d623278cae8e9bdf134d06700e4c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185714716389Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84632001536, + "active_bytes_current_delta": 41078656512, + "active_bytes_peak": 91613566976, + "active_bytes_peak_delta": 48053930496, + "after": { + "active_bytes_current": 84632001536, + "active_bytes_peak": 91613566976, + "allocated_bytes_current": 84632001536, + "allocated_bytes_peak": 91613566976, + "device": 1, + "inactive_split_bytes_current": 4169802752, + "inactive_split_bytes_peak": 12200414208, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 92612329472, + "reserved_bytes_peak": 92612329472, + "segment_count_current": 2204, + "segment_count_peak": 2204, + "timestamp_utc": "2026-08-26T18:57:14.715908Z", + "world_size": 8 + }, + "allocated_bytes_current": 84632001536, + "allocated_bytes_current_delta": 41078656512, + "allocated_bytes_peak": 91613566976, + "allocated_bytes_peak_delta": 48053930496, + "before": { + "active_bytes_current": 43553345024, + "active_bytes_peak": 43559636480, + "allocated_bytes_current": 43553345024, + "allocated_bytes_peak": 43559636480, + "device": 1, + "inactive_split_bytes_current": 1254404608, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:56:36.124275Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4169802752, + "inactive_split_bytes_current_delta": 2915398144, + "inactive_split_bytes_peak": 12200414208, + "inactive_split_bytes_peak_delta": 10587501568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 92612329472, + "reserved_bytes_current_delta": 47787802624, + "reserved_bytes_peak": 92612329472, + "reserved_bytes_peak_delta": 47787802624, + "segment_count_current": 2204, + "segment_count_current_delta": 1314, + "segment_count_peak": 2204, + "segment_count_peak_delta": 1314, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185714716389Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.908307Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185714716389Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185714716389Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0642e326ffa484aa9f6fb8f839d5fd258311d81 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185714716389Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:169bbd0bc7fca26917ea532af35bdc82852861d3313c803ef388fab4fc90d081 +size 9743823 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185716268577Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185716268577Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78d3ca0418a0a1b977c8d9024af8d1805ae1fffd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185716268577Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84632001536, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122395896320, + "active_bytes_peak_delta": 30782329344, + "after": { + "active_bytes_current": 84632001536, + "active_bytes_peak": 122395896320, + "allocated_bytes_current": 84632001536, + "allocated_bytes_peak": 122395896320, + "device": 1, + "inactive_split_bytes_current": 4071236608, + "inactive_split_bytes_peak": 12200414208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 123394326528, + "reserved_bytes_peak": 123394326528, + "segment_count_current": 2923, + "segment_count_peak": 2923, + "timestamp_utc": "2026-08-26T18:57:16.268143Z", + "world_size": 8 + }, + "allocated_bytes_current": 84632001536, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122395896320, + "allocated_bytes_peak_delta": 30782329344, + "before": { + "active_bytes_current": 84632001536, + "active_bytes_peak": 91613566976, + "allocated_bytes_current": 84632001536, + "allocated_bytes_peak": 91613566976, + "device": 1, + "inactive_split_bytes_current": 4169802752, + "inactive_split_bytes_peak": 12200414208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 92612329472, + "reserved_bytes_peak": 92612329472, + "segment_count_current": 2204, + "segment_count_peak": 2204, + "timestamp_utc": "2026-08-26T18:57:14.945993Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4071236608, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12200414208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 123394326528, + "reserved_bytes_current_delta": 30781997056, + "reserved_bytes_peak": 123394326528, + "reserved_bytes_peak_delta": 30781997056, + "segment_count_current": 2923, + "segment_count_current_delta": 719, + "segment_count_peak": 2923, + "segment_count_peak_delta": 719, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185716268577Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.462608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185716268577Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185716268577Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..770a2bfb955c681b46f0fa53f6c40baef638ab4e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185716268577Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:028279b77877790e43e380310ceb3d30a01c9b800623f2bb1999c8522d65d705 +size 9963530 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1049714bc9c37e8496f18839154547b92bbf854e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84632001536, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122408459264, + "active_bytes_peak_delta": 12562944, + "after": { + "active_bytes_current": 84632001536, + "active_bytes_peak": 122408459264, + "allocated_bytes_current": 84632001536, + "allocated_bytes_peak": 122408459264, + "device": 1, + "inactive_split_bytes_current": 4173997056, + "inactive_split_bytes_peak": 12200414208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 123394326528, + "reserved_bytes_peak": 123394326528, + "segment_count_current": 2923, + "segment_count_peak": 2923, + "timestamp_utc": "2026-08-26T18:57:17.636833Z", + "world_size": 8 + }, + "allocated_bytes_current": 84632001536, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122408459264, + "allocated_bytes_peak_delta": 12562944, + "before": { + "active_bytes_current": 84632001536, + "active_bytes_peak": 122395896320, + "allocated_bytes_current": 84632001536, + "allocated_bytes_peak": 122395896320, + "device": 1, + "inactive_split_bytes_current": 4071236608, + "inactive_split_bytes_peak": 12200414208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 123394326528, + "reserved_bytes_peak": 123394326528, + "segment_count_current": 2923, + "segment_count_peak": 2923, + "timestamp_utc": "2026-08-26T18:57:16.490751Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4173997056, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 12200414208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 123394326528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123394326528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2923, + "segment_count_current_delta": 0, + "segment_count_peak": 2923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.834858Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file