diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv index 1c40b9a259e01820666df8583b52420d88c30e15..1f462276b4449e5f1cb9167cca7aca94d60ed7a2 100644 --- a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -94,3 +94,155 @@ 5, 54107 6, 54107 7, 54107 +0, 54479 +1, 54497 +2, 54469 +3, 54483 +4, 54191 +5, 54499 +6, 54473 +7, 54487 +0, 55545 +1, 55545 +2, 55545 +3, 55545 +4, 55257 +5, 55545 +6, 55545 +7, 55545 +0, 56057 +1, 56057 +2, 56057 +3, 56057 +4, 55769 +5, 56057 +6, 56057 +7, 56057 +0, 56971 +1, 56971 +2, 56971 +3, 56971 +4, 56683 +5, 56971 +6, 56971 +7, 56971 +0, 57593 +1, 57591 +2, 57591 +3, 57591 +4, 57305 +5, 57591 +6, 57591 +7, 57591 +0, 96129 +1, 94601 +2, 94601 +3, 94601 +4, 94573 +5, 94601 +6, 96173 +7, 94601 +0, 96841 +1, 96885 +2, 96885 +3, 96885 +4, 96857 +5, 96885 +6, 96885 +7, 96885 +0, 96841 +1, 96885 +2, 96885 +3, 96885 +4, 96857 +5, 96885 +6, 96885 +7, 96885 +0, 96841 +1, 96885 +2, 96885 +3, 96885 +4, 96857 +5, 96885 +6, 96885 +7, 96885 +0, 96841 +1, 96885 +2, 96885 +3, 96885 +4, 96857 +5, 96885 +6, 96885 +7, 96885 +0, 96841 +1, 96885 +2, 96885 +3, 96885 +4, 96857 +5, 96885 +6, 96885 +7, 96885 +0, 98191 +1, 98235 +2, 98235 +3, 98235 +4, 98207 +5, 98235 +6, 98235 +7, 98235 +0, 101927 +1, 101971 +2, 101971 +3, 101971 +4, 101943 +5, 101971 +6, 101971 +7, 101971 +0, 108281 +1, 108393 +2, 108349 +3, 108345 +4, 108419 +5, 108345 +6, 108391 +7, 108403 +0, 120723 +1, 120415 +2, 120335 +3, 119923 +4, 119591 +5, 120455 +6, 120755 +7, 119891 +0, 145005 +1, 145199 +2, 145199 +3, 145199 +4, 145347 +5, 145199 +6, 145199 +7, 145199 +0, 143759 +1, 143953 +2, 143953 +3, 143953 +4, 144165 +5, 144599 +6, 144273 +7, 143953 +0, 4643 +1, 3366 +2, 3366 +3, 3366 +4, 3366 +5, 3366 +6, 3622 +7, 143951 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea921e0d36853d735a2ec1c6f51c9ca0819dd11f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85292954112, + "active_bytes_current_delta": 40772344320, + "active_bytes_peak": 90610880000, + "active_bytes_peak_delta": 46065104384, + "after": { + "active_bytes_current": 85292954112, + "active_bytes_peak": 90610880000, + "allocated_bytes_current": 85292954112, + "allocated_bytes_peak": 90610880000, + "device": 0, + "inactive_split_bytes_current": 1365560832, + "inactive_split_bytes_peak": 11201188352, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 103173586944, + "reserved_bytes_peak": 103173586944, + "segment_count_current": 1786, + "segment_count_peak": 1786, + "timestamp_utc": "2026-08-26T18:50:34.927898Z", + "world_size": 8 + }, + "allocated_bytes_current": 85292954112, + "allocated_bytes_current_delta": 40772344320, + "allocated_bytes_peak": 90610880000, + "allocated_bytes_peak_delta": 46065104384, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 0, + "inactive_split_bytes_current": 446523392, + "inactive_split_bytes_peak": 1050503168, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:49:54.861513Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1365560832, + "inactive_split_bytes_current_delta": 919037440, + "inactive_split_bytes_peak": 11201188352, + "inactive_split_bytes_peak_delta": 10150685184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 103173586944, + "reserved_bytes_current_delta": 54922313728, + "reserved_bytes_peak": 103173586944, + "reserved_bytes_peak_delta": 54922313728, + "segment_count_current": 1786, + "segment_count_current_delta": 1165, + "segment_count_peak": 1786, + "segment_count_peak_delta": 1165, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.981197Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb8783b94d61333587102db9cd9be4c3cbae3391 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185034928355Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aa4a4bb58a42d4afb2348619f36704949c1954e13d6810a889605bef2a6594a +size 3160743 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6905be7e28d0745c4bf894291c49bd737cddabf5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85292954112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477026816, + "active_bytes_peak_delta": 30866146816, + "after": { + "active_bytes_current": 85292954112, + "active_bytes_peak": 121477026816, + "allocated_bytes_current": 85292954112, + "allocated_bytes_peak": 121477026816, + "device": 0, + "inactive_split_bytes_current": 1770311168, + "inactive_split_bytes_peak": 11201188352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 128687538176, + "reserved_bytes_peak": 128687538176, + "segment_count_current": 2005, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:35.546179Z", + "world_size": 8 + }, + "allocated_bytes_current": 85292954112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121477026816, + "allocated_bytes_peak_delta": 30866146816, + "before": { + "active_bytes_current": 85292954112, + "active_bytes_peak": 90610880000, + "allocated_bytes_current": 85292954112, + "allocated_bytes_peak": 90610880000, + "device": 0, + "inactive_split_bytes_current": 1365560832, + "inactive_split_bytes_peak": 11201188352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 103173586944, + "reserved_bytes_peak": 103173586944, + "segment_count_current": 1786, + "segment_count_peak": 1786, + "timestamp_utc": "2026-08-26T18:50:34.987192Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1770311168, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 11201188352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 128687538176, + "reserved_bytes_current_delta": 25513951232, + "reserved_bytes_peak": 128687538176, + "reserved_bytes_peak_delta": 25513951232, + "segment_count_current": 2005, + "segment_count_current_delta": 219, + "segment_count_peak": 2005, + "segment_count_peak_delta": 219, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.595543Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3552f5e8c1b1d1e6f2401fe9037133f30fed3b2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185035546604Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2ab06c8c945ecdb672016c8f54cb38720c97c1653b13de54badb3b299e51a4 +size 2586991 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f3862ea9671fc2956c7bd7e5bd36cf4c000fe9a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85292954112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477075968, + "active_bytes_peak_delta": 49152, + "after": { + "active_bytes_current": 85292954112, + "active_bytes_peak": 121477075968, + "allocated_bytes_current": 85292954112, + "allocated_bytes_peak": 121477075968, + "device": 0, + "inactive_split_bytes_current": 1770311168, + "inactive_split_bytes_peak": 11201188352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 128687538176, + "reserved_bytes_peak": 128687538176, + "segment_count_current": 2005, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:35.971739Z", + "world_size": 8 + }, + "allocated_bytes_current": 85292954112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121477075968, + "allocated_bytes_peak_delta": 49152, + "before": { + "active_bytes_current": 85292954112, + "active_bytes_peak": 121477026816, + "allocated_bytes_current": 85292954112, + "allocated_bytes_peak": 121477026816, + "device": 0, + "inactive_split_bytes_current": 1770311168, + "inactive_split_bytes_peak": 11201188352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 128687538176, + "reserved_bytes_peak": 128687538176, + "segment_count_current": 2005, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:35.601397Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1770311168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11201188352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 128687538176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128687538176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2005, + "segment_count_current_delta": 0, + "segment_count_peak": 2005, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.017227Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94a9ac50485aa207511799195d4f6982f7d498a2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185035972144Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e91cae5e5b40799ac6bd86fc41b310aa28ee2e4b11be02b9bd4055f177b4cb5 +size 2609276 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.json new file mode 100644 index 0000000000000000000000000000000000000000..386aba5c8dae8980c63b9e034be64830ad2afe2d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85309997568, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128733144064, + "active_bytes_peak_delta": 7256068096, + "after": { + "active_bytes_current": 85309997568, + "active_bytes_peak": 128733144064, + "allocated_bytes_current": 85309997568, + "allocated_bytes_peak": 121485386752, + "device": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_peak": 12882581504, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_peak": 141482262528, + "segment_count_current": 1805, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:38.956995Z", + "world_size": 8 + }, + "allocated_bytes_current": 85309997568, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121485386752, + "allocated_bytes_peak_delta": 8310784, + "before": { + "active_bytes_current": 85292954112, + "active_bytes_peak": 121477075968, + "allocated_bytes_current": 85292954112, + "allocated_bytes_peak": 121477075968, + "device": 0, + "inactive_split_bytes_current": 1770311168, + "inactive_split_bytes_peak": 11201188352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 128687538176, + "reserved_bytes_peak": 128687538176, + "segment_count_current": 2005, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:36.022500Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_current_delta": -316936192, + "inactive_split_bytes_peak": 12882581504, + "inactive_split_bytes_peak_delta": 1681393152, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_current_delta": 12794724352, + "reserved_bytes_peak": 141482262528, + "reserved_bytes_peak_delta": 12794724352, + "segment_count_current": 1805, + "segment_count_current_delta": -200, + "segment_count_peak": 2005, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.010451Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1512796ffe1af14a63b3d74e01af62607f0c48e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185038957549Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e3a5548b896b2a25ca4a2fb6b2899d86e34f93bf80fffcbbb54656f939a519c +size 2779026 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5f5bb822f0b4fbc82ecaaaa299ddd0fb5f55d5a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85309997568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128733144064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85309997568, + "active_bytes_peak": 128733144064, + "allocated_bytes_current": 85309997568, + "allocated_bytes_peak": 121485386752, + "device": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_peak": 12882581504, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_peak": 141482262528, + "segment_count_current": 1805, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:39.331583Z", + "world_size": 8 + }, + "allocated_bytes_current": 85309997568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121485386752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85309997568, + "active_bytes_peak": 128733144064, + "allocated_bytes_current": 85309997568, + "allocated_bytes_peak": 121485386752, + "device": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_peak": 12882581504, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_peak": 141482262528, + "segment_count_current": 1805, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:39.017489Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12882581504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141482262528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1805, + "segment_count_current_delta": 0, + "segment_count_peak": 2005, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.342045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbe67f8e60946f5176fbb37f5117e0d79cbbaafe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185039332056Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6183e257ea248a054b02716706577af577188c192e256577c3ffa6d94ccb4de7 +size 422083 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6968aa8c79f3b6b18ece79a25149a0b987bcd228 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85309997568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128733144064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85309997568, + "active_bytes_peak": 128733144064, + "allocated_bytes_current": 85309997568, + "allocated_bytes_peak": 121485386752, + "device": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_peak": 12882581504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_peak": 141482262528, + "segment_count_current": 1805, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:39.652667Z", + "world_size": 8 + }, + "allocated_bytes_current": 85309997568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121485386752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85309997568, + "active_bytes_peak": 128733144064, + "allocated_bytes_current": 85309997568, + "allocated_bytes_peak": 121485386752, + "device": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_peak": 12882581504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_peak": 141482262528, + "segment_count_current": 1805, + "segment_count_peak": 2005, + "timestamp_utc": "2026-08-26T18:50:39.343193Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1453374976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12882581504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 141482262528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141482262528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1805, + "segment_count_current_delta": 0, + "segment_count_peak": 2005, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.662359Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29f9f8fca4d6e01151f6a8be236e5b34631e1b3d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185039653043Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bd61744ce6d87e4c745b1d183902c4d00ccc1bd8a56aad6e8a91430ad96fbb6 +size 422083 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d0fd4bc895d30d9c62beab9ef14be0abcfba63d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 40774371840, + "active_bytes_peak": 90612776448, + "active_bytes_peak_delta": 46067000832, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 1, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.927803Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 40774371840, + "allocated_bytes_peak": 90612776448, + "allocated_bytes_peak_delta": 46067000832, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 1, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:49:54.861521Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_current_delta": 1145599488, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 10888034816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 103244890112, + "reserved_bytes_current_delta": 54993616896, + "reserved_bytes_peak": 103244890112, + "reserved_bytes_peak_delta": 54993616896, + "segment_count_current": 1790, + "segment_count_current_delta": 1169, + "segment_count_peak": 1790, + "segment_count_peak_delta": 1169, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.980104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa9ba53ae8f6f511a5b99aa2b0562bfa7fe02545 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185034928307Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd97ae4b71d75d8df2690b1b9bd0903c3090a75cbc027e10b301f7c4dccd2865 +size 3148002 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78ed9366fe216aff8568062c9d8c279676f8a811 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478210048, + "active_bytes_peak_delta": 30865433600, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 1, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.546159Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478210048, + "allocated_bytes_peak_delta": 30865433600, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 1, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.986365Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 25560088576, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 25560088576, + "segment_count_current": 2012, + "segment_count_current_delta": 222, + "segment_count_peak": 2012, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.596037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98b9672c611df2b569c72733307a67c752e12410 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185035546572Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af77a838b6c6a61374c9704ed59f260069beb78734595ba0949d7d8f173b1c0b +size 2579979 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1846b078721d5b0811e6c4c50186a58880301b53 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478455808, + "active_bytes_peak_delta": 245760, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 1, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.971660Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478455808, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 1, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.602405Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2012, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.017792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee6a965abe3cfd39fa4faed2d7193c082415598a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185035972066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8705912aae623c560ae0d7d576d6e67e34ff869f0e694974c17f0d2cccc07fd0 +size 2569019 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f7488b597eaa4c3453adce5700698edfac775869 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 7256715776, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 1, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:38.956984Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 8958464, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 1, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:36.022865Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": -319033344, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 1170605568, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 12880707584, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 12880707584, + "segment_count_current": 1815, + "segment_count_current_delta": -197, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.010026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5d1259e767537ef3ecae6dde337fa4a0012c517 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185038957519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29bc1811482194f01ba57189d5b0336f0a556b13bcd06f639573a807a68f2d02 +size 2771672 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.json new file mode 100644 index 0000000000000000000000000000000000000000..059444d66aa8ebf2a6f3f80cb729bcfa3a51df7c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 1, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.331461Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 1, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.015933Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.341247Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..540ecbbd045981149066debfb59526fceb590014 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185039331856Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f31fa3e47c17b389551300bd90296f5b7d4a52bd842eebe30fe68356467f07a0 +size 416145 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72f640311a33111ac6713b836ad6fe81723ab966 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 1, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.652649Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 1, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.342351Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.662081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da4345137c085502f46cfa2fe2961a35417c86ea --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185039653028Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52f33d0f617a93d8d768f32da49c5a936943607e881cc1a361ba3301d7d47d13 +size 416145 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9fccc1243b03eb970a06f34be5bedf485361eb1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 40774371840, + "active_bytes_peak": 90612776448, + "active_bytes_peak_delta": 46067000832, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 2, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.927806Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 40774371840, + "allocated_bytes_peak": 90612776448, + "allocated_bytes_peak_delta": 46067000832, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 2, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:49:54.861483Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_current_delta": 1145599488, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 10888034816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 103244890112, + "reserved_bytes_current_delta": 54993616896, + "reserved_bytes_peak": 103244890112, + "reserved_bytes_peak_delta": 54993616896, + "segment_count_current": 1790, + "segment_count_current_delta": 1169, + "segment_count_peak": 1790, + "segment_count_peak_delta": 1169, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.981556Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b591baf8f254e7a6b5e8c5b7c8bf786730923e35 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185034928309Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27b146b2dfaeba3efc68604c5b3c53fb4d0bbb10f8c3a1b64518f0b3c3060ef1 +size 3148002 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ba85cda5c547cccb04419b94baaca9aabc0c63a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478210048, + "active_bytes_peak_delta": 30865433600, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 2, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.546227Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478210048, + "allocated_bytes_peak_delta": 30865433600, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 2, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.987886Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 25560088576, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 25560088576, + "segment_count_current": 2012, + "segment_count_current_delta": 222, + "segment_count_peak": 2012, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.595565Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf7cbcdb68d0f15a3c3a17f4f0e89c4838d3e5ee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185035546627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5b1e78a390fe971af9f4fc30d95e219aac098eba527b9f930025a3321f93456 +size 2579948 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d729cbc10e51988023c840fa78bcb6b8b467961 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478455808, + "active_bytes_peak_delta": 245760, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 2, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.971658Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478455808, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 2, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.601963Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2012, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.016491Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c59a8baa1ed66ef350e0a84762c81495f72a074 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185035972065Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:249e9b8f6d474565c05df97910be1857323ef733f2bf81f86f2e60db2e0c0a4f +size 2569055 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dcbde9bc2f1dee5b0d3caca7d1396b3061b2a1ba --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 7256715776, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 2, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:38.956987Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 8958464, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 2, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:36.021547Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": -319033344, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 1170605568, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 12880707584, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 12880707584, + "segment_count_current": 1815, + "segment_count_current_delta": -197, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.010224Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf67f22ea3c4154011b5ec9a6e535888cfc9e7a5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185038957509Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4861d2922ba964c05d12e06573ce96afcd744654d2763ee57e9b0332af50b0e9 +size 2771672 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82a2d0d9936662b88e3a05876256cbf9139f023c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 2, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.331456Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 2, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.016318Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.341347Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b163ffc9ea1b9a739ff4630f39dee820fdb8116 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185039331849Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91b5d7b1733ad80c6a16d831fe2407c6b08974fd6fc2484f9afd62e0a6f0c9d0 +size 416145 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1ace93a0de69eb7502c14c3ce5ccee20786408b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 2, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.652654Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 2, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.342440Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.662013Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7684bb774a7cfccbda0276b0a97e1191714df4c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185039653029Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48bea32c141f6c0dc478293871003f1d61d30ebf5a6daf2e6748d169924c2102 +size 416145 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f4e9bdce6c82eabdfbd464e54ad026597e76245 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 40774371840, + "active_bytes_peak": 90612776448, + "active_bytes_peak_delta": 46067000832, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 3, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.927781Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 40774371840, + "allocated_bytes_peak": 90612776448, + "allocated_bytes_peak_delta": 46067000832, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 3, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:49:54.861469Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_current_delta": 1145599488, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 10888034816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 103244890112, + "reserved_bytes_current_delta": 54993616896, + "reserved_bytes_peak": 103244890112, + "reserved_bytes_peak_delta": 54993616896, + "segment_count_current": 1790, + "segment_count_current_delta": 1169, + "segment_count_peak": 1790, + "segment_count_peak_delta": 1169, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.979803Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8afad3fa8f9f4da5359b0e4172575f0ec260197e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185034928293Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bd36093f315ca9b352532172956de6dfd4183b9176483c7a47e15c64fba50af +size 3147999 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9bbddc53d8c0f557c3dff37caefd2dc01622152 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478210048, + "active_bytes_peak_delta": 30865433600, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 3, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.546147Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478210048, + "allocated_bytes_peak_delta": 30865433600, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 3, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.986030Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 25560088576, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 25560088576, + "segment_count_current": 2012, + "segment_count_current_delta": 222, + "segment_count_peak": 2012, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.594867Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3932eb0c08666bcf894db1565ba2b17f3aa0e92f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185035546567Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e587809f83fdda618266730bd28e2282bd753e40bc02169039a8e4b34ceaf25d +size 2579979 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76434e5fc2941047b46078316b918f0ccf9340d3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122082435584, + "active_bytes_peak_delta": 604225536, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 122082435584, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 3, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.971654Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478455808, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 3, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.601253Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2012, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.016411Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1823c965b9ca690f7241d8734c498fde62bc076 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185035972066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ebc496a7c1c0b713c075ec11ebed6a4aca0be3a6c15bf3abbe78aafc67ca8b8 +size 2569019 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b39642e8409d6e160474e8d7ccd279be9b2d03d2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 6652736000, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 3, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12815542272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:38.956989Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 8958464, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 122082435584, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 3, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:36.021568Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": -319033344, + "inactive_split_bytes_peak": 12815542272, + "inactive_split_bytes_peak_delta": 902170112, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 12880707584, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 12880707584, + "segment_count_current": 1815, + "segment_count_current_delta": -197, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.010626Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41908096b401085d4e0b60e6e4ac4821e016594d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185038957534Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e19767dc90ece8d439f6886a341dd416885067ba2615502ae69deab56959ca22 +size 2771669 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2d1beb7d9f0620cd8f94adee870975cc9e3c16d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 3, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12815542272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.331469Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 3, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12815542272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.016682Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12815542272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.341608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93d86b03c79c1554656d3e27aa6bbbc778ac86ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185039331856Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ba49380f0f1c7c224d6180fa854e53bfa52c31ea95d95b166dfceece6a8f2e2 +size 416035 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d3739b643dc58d5c9f6c9c58e7a267b5c0646bf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 3, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12815542272, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.652658Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 3, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12815542272, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.342717Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12815542272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.662417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f2c49b001ed4352beef6d736c3f635c527f460a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185039653038Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77688a743792e2b71ed5652b03f4fb807657b5a012cf454a3c1eceb47d63c720 +size 416035 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fe7b5a2a130a03d31b8b1022089961562ad6b32 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295019520, + "active_bytes_current_delta": 40774409728, + "active_bytes_peak": 90623401472, + "active_bytes_peak_delta": 46077625856, + "after": { + "active_bytes_current": 85295019520, + "active_bytes_peak": 90623401472, + "allocated_bytes_current": 85295019520, + "allocated_bytes_peak": 90623401472, + "device": 4, + "inactive_split_bytes_current": 1720011264, + "inactive_split_bytes_peak": 11677395968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 103291027456, + "reserved_bytes_peak": 103291027456, + "segment_count_current": 1798, + "segment_count_peak": 1798, + "timestamp_utc": "2026-08-26T18:50:34.927754Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295019520, + "allocated_bytes_current_delta": 40774409728, + "allocated_bytes_peak": 90623401472, + "allocated_bytes_peak_delta": 46077625856, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 4, + "inactive_split_bytes_current": 270362624, + "inactive_split_bytes_peak": 874342400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T18:49:54.861438Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1720011264, + "inactive_split_bytes_current_delta": 1449648640, + "inactive_split_bytes_peak": 11677395968, + "inactive_split_bytes_peak_delta": 10803053568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 103291027456, + "reserved_bytes_current_delta": 55341744128, + "reserved_bytes_peak": 103291027456, + "reserved_bytes_peak_delta": 55341744128, + "segment_count_current": 1798, + "segment_count_current_delta": 1183, + "segment_count_peak": 1798, + "segment_count_peak_delta": 1183, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.981424Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96abe4b2a4a09f4e523f90973becb0ccaa3cd369 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185034928303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:096e53d0d4bcb3bea8c7c5a73d3ca4779d6e6aa4d36309b0956df64fce72f02f +size 3156436 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185035546628Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185035546628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be8885cdccb9c0c9f2d0ac79df1807ea29bf29ae --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185035546628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295019520, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121518197760, + "active_bytes_peak_delta": 30894796288, + "after": { + "active_bytes_current": 85295019520, + "active_bytes_peak": 121518197760, + "allocated_bytes_current": 85295019520, + "allocated_bytes_peak": 121518197760, + "device": 4, + "inactive_split_bytes_current": 2126858752, + "inactive_split_bytes_peak": 11677395968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2017, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:35.546205Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295019520, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121518197760, + "allocated_bytes_peak_delta": 30894796288, + "before": { + "active_bytes_current": 85295019520, + "active_bytes_peak": 90623401472, + "allocated_bytes_current": 85295019520, + "allocated_bytes_peak": 90623401472, + "device": 4, + "inactive_split_bytes_current": 1720011264, + "inactive_split_bytes_peak": 11677395968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 103291027456, + "reserved_bytes_peak": 103291027456, + "segment_count_current": 1798, + "segment_count_peak": 1798, + "timestamp_utc": "2026-08-26T18:50:34.987216Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2126858752, + "inactive_split_bytes_current_delta": 406847488, + "inactive_split_bytes_peak": 11677395968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 25513951232, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 25513951232, + "segment_count_current": 2017, + "segment_count_current_delta": 219, + "segment_count_peak": 2017, + "segment_count_peak_delta": 219, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185035546628Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.597011Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185035546628Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185035546628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca73157e371ae8e9a2e30683e94c5b49b910a5a9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185035546628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb6d6dab131dced5a82d23f9adbc9d8be889c2b25a105b357031bd87efba0896 +size 2552206 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185035972067Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185035972067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03badbdafdf1961fbce5d1cb93cd2f702278960b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185035972067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295019520, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121522668544, + "active_bytes_peak_delta": 4470784, + "after": { + "active_bytes_current": 85295019520, + "active_bytes_peak": 121522668544, + "allocated_bytes_current": 85295019520, + "allocated_bytes_peak": 121522668544, + "device": 4, + "inactive_split_bytes_current": 2126858752, + "inactive_split_bytes_peak": 11677395968, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2017, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:35.971655Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295019520, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121522668544, + "allocated_bytes_peak_delta": 4470784, + "before": { + "active_bytes_current": 85295019520, + "active_bytes_peak": 121518197760, + "allocated_bytes_current": 85295019520, + "allocated_bytes_peak": 121518197760, + "device": 4, + "inactive_split_bytes_current": 2126858752, + "inactive_split_bytes_peak": 11677395968, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2017, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:35.602805Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2126858752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11677395968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2017, + "segment_count_current_delta": 0, + "segment_count_peak": 2017, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185035972067Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.017893Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185035972067Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185035972067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11571b202f7a3e7fd3cf21f8b6822f6dd2eb3b01 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185035972067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06d4213f28ce2651160e336fe0092dedf936e0a4995a0f6010f9f0f073efba9f +size 2647194 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185038957549Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185038957549Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3a96d30fb797eb06cff72592d8f42a274512479 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185038957549Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312062976, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128735209472, + "active_bytes_peak_delta": 7212540928, + "after": { + "active_bytes_current": 85312062976, + "active_bytes_peak": 128735209472, + "allocated_bytes_current": 85312062976, + "allocated_bytes_peak": 121522668544, + "device": 4, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_peak": 13239129088, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141840875520, + "reserved_bytes_peak": 141840875520, + "segment_count_current": 1841, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:38.956985Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312062976, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121522668544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85295019520, + "active_bytes_peak": 121522668544, + "allocated_bytes_current": 85295019520, + "allocated_bytes_peak": 121522668544, + "device": 4, + "inactive_split_bytes_current": 2126858752, + "inactive_split_bytes_peak": 11677395968, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2017, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:36.023081Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_current_delta": -319033344, + "inactive_split_bytes_peak": 13239129088, + "inactive_split_bytes_peak_delta": 1561733120, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141840875520, + "reserved_bytes_current_delta": 13035896832, + "reserved_bytes_peak": 141840875520, + "reserved_bytes_peak_delta": 13035896832, + "segment_count_current": 1841, + "segment_count_current_delta": -176, + "segment_count_peak": 2017, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185038957549Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.012217Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185038957549Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185038957549Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8190ea859e2227b14b74adb3520ec096b588380f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185038957549Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:041888524c6d7966730693a18b60097c5fc43fc51c9043cee5e61385f7adae0d +size 3530395 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185039331871Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185039331871Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1522d6d2ce5e784dbaa38bccfa33a20feff23f3a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185039331871Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312062976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735209472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312062976, + "active_bytes_peak": 128735209472, + "allocated_bytes_current": 85312062976, + "allocated_bytes_peak": 121522668544, + "device": 4, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_peak": 13239129088, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141840875520, + "reserved_bytes_peak": 141840875520, + "segment_count_current": 1841, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:39.331446Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312062976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121522668544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312062976, + "active_bytes_peak": 128735209472, + "allocated_bytes_current": 85312062976, + "allocated_bytes_peak": 121522668544, + "device": 4, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_peak": 13239129088, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141840875520, + "reserved_bytes_peak": 141840875520, + "segment_count_current": 1841, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:39.018323Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13239129088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141840875520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141840875520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1841, + "segment_count_current_delta": 0, + "segment_count_peak": 2017, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185039331871Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.341653Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185039331871Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185039331871Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..159bc5f06f367327440635c411e62f9e2d4f7fbe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185039331871Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55d8f45ee912bd059595ea649d742c9007f593597aa7ee41a99d56651ba6e896 +size 485218 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185039653026Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185039653026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eea56ace6858b4372bc887f8acd601b08989e41e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185039653026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312062976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735209472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312062976, + "active_bytes_peak": 128735209472, + "allocated_bytes_current": 85312062976, + "allocated_bytes_peak": 121522668544, + "device": 4, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_peak": 13239129088, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141840875520, + "reserved_bytes_peak": 141840875520, + "segment_count_current": 1841, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:39.652631Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312062976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121522668544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312062976, + "active_bytes_peak": 128735209472, + "allocated_bytes_current": 85312062976, + "allocated_bytes_peak": 121522668544, + "device": 4, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_peak": 13239129088, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141840875520, + "reserved_bytes_peak": 141840875520, + "segment_count_current": 1841, + "segment_count_peak": 2017, + "timestamp_utc": "2026-08-26T18:50:39.342790Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1807825408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13239129088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141840875520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141840875520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1841, + "segment_count_current_delta": 0, + "segment_count_peak": 2017, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185039653026Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.662436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185039653026Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185039653026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f271437117201ce37fe93bc10bef4072727100c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185039653026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d11de8fdac08c4a89ed20a275f222cca087fcdb995cbef3dde6b386c3250d631 +size 485187 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185034928307Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185034928307Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e5c2ae6b033ae755793ca10ca1cdd1cb6500cc6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185034928307Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 40774371840, + "active_bytes_peak": 90612776448, + "active_bytes_peak_delta": 46067000832, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 5, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.927848Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 40774371840, + "allocated_bytes_peak": 90612776448, + "allocated_bytes_peak_delta": 46067000832, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 5, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:49:54.861454Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_current_delta": 1145599488, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 10888034816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 103244890112, + "reserved_bytes_current_delta": 54993616896, + "reserved_bytes_peak": 103244890112, + "reserved_bytes_peak_delta": 54993616896, + "segment_count_current": 1790, + "segment_count_current_delta": 1169, + "segment_count_peak": 1790, + "segment_count_peak_delta": 1169, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185034928307Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.981017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185034928307Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185034928307Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a1886552938b577ee786c8192551a73ba938818 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185034928307Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24b2f211a33d3eb39452bb31d8bbb2794b5b6e5d5d853d20fe7108773d5e9ae1 +size 3148002 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185035546624Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185035546624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b316f38896e2c53e5201903661208eb97e09339 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185035546624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478210048, + "active_bytes_peak_delta": 30865433600, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 5, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.546192Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478210048, + "allocated_bytes_peak_delta": 30865433600, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 5, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.987164Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 25560088576, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 25560088576, + "segment_count_current": 2012, + "segment_count_current_delta": 222, + "segment_count_peak": 2012, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185035546624Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.600433Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185035546624Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185035546624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b30043e725c7b3a81c624128e142798250b70a1a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185035546624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e25574da30b680153edd1a3f11fcc80a195567616234bd689f7d6156bc8acf +size 2579985 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185035972235Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185035972235Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3ac16698767324012aad5ec6b0a241760ae48ef --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185035972235Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478455808, + "active_bytes_peak_delta": 245760, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 5, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.971779Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478455808, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 5, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.606741Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2012, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185035972235Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.019571Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185035972235Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185035972235Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..842755bc30a6db2d132f91d45dd03df37fb96890 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185035972235Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36cd246dde3c952652653a28e7f3f098bb6447533163986ed52e7e4dbb669934 +size 2569019 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185038957570Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185038957570Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42f8ba49afbfd37d5e2f9e42735aae262ec8ca4b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185038957570Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 7256715776, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 5, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:38.957018Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 8958464, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 5, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:36.024908Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": -319033344, + "inactive_split_bytes_peak": 12988620288, + "inactive_split_bytes_peak_delta": 1075248128, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 12880707584, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 12880707584, + "segment_count_current": 1815, + "segment_count_current_delta": -197, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185038957570Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.010109Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185038957570Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185038957570Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..000666841ac0c627f9b875c6166efaf2194543e2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185038957570Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aac59d478d2f38b66c4affbf52633e05b245185424613ca52b41ca4e5117fa18 +size 2771672 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185039331861Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185039331861Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2400b88d4ef908f5bdc17173716748b6846f9467 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185039331861Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 5, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.331470Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 5, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.015998Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12988620288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185039331861Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.341108Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185039331861Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185039331861Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..abc4069b067446e8708fdd18b1d3c652ab79cbdd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185039331861Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3e3b64afda9df001f65f8900d73be575f3318b0451d641fd10d06521dc30845 +size 416145 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185039653071Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185039653071Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1a25696942a314a19cd2b4b927a5c6c70121e4f4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185039653071Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 5, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.652685Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 5, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.342231Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12988620288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185039653071Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.662794Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185039653071Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185039653071Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b721087bf1e24a859dffd451009912980e64841c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185039653071Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa48a89fe34d77ddb0a778421221564ce9c5aba141fc610e9918b99cccc87551 +size 416145 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185034928329Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185034928329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..074a64bf499ec73133997c83d0bcd1e522211948 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185034928329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 40774371840, + "active_bytes_peak": 90612776448, + "active_bytes_peak_delta": 46067000832, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 6, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.927870Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 40774371840, + "allocated_bytes_peak": 90612776448, + "allocated_bytes_peak_delta": 46067000832, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 6, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:49:54.861476Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_current_delta": 1145599488, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 10888034816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 103244890112, + "reserved_bytes_current_delta": 54993616896, + "reserved_bytes_peak": 103244890112, + "reserved_bytes_peak_delta": 54993616896, + "segment_count_current": 1790, + "segment_count_current_delta": 1169, + "segment_count_peak": 1790, + "segment_count_peak_delta": 1169, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185034928329Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.980681Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185034928329Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185034928329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80bc15dda8d9bf07277c8d89e61fc56b5dbf1327 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185034928329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29ecd49995f840b118a7bb679c83f54e2d45a82da14ae8f402e47ee10e67af72 +size 3147996 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185035546603Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185035546603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f818567d5b422dc4c93297fbc8433cfa3293c9bf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185035546603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478210048, + "active_bytes_peak_delta": 30865433600, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 6, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.546171Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478210048, + "allocated_bytes_peak_delta": 30865433600, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 6, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.986886Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 25560088576, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 25560088576, + "segment_count_current": 2012, + "segment_count_current_delta": 222, + "segment_count_peak": 2012, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185035546603Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.594865Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185035546603Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185035546603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8bb5dbaa77cdb6ce99c65c50cbc3cbedd257c27b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185035546603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b351cf42d2ede85c82b2bcfbe001fe3a63a455a911283040382c5daeaa395dc +size 2583976 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185035972216Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185035972216Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a139df33fdee2909b1210d7fd8def0e8420c7ba7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185035972216Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122082435584, + "active_bytes_peak_delta": 604225536, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 122082435584, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 6, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.971771Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478455808, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 6, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.600889Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2012, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185035972216Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.019344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185035972216Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185035972216Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b320a10538bde976b419652bfdb778d19751cf1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185035972216Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:102902cc2c483e2b1911ef6e9a5d9d98fad9225bd3765fea24c32677ef68fad9 +size 2569002 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185038957530Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185038957530Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d89b3b8da2099053aa71918c50b377f06785a135 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185038957530Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 6652736000, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 6, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:38.957012Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 8958464, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 122082435584, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 6, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:36.024691Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": -319033344, + "inactive_split_bytes_peak": 12988620288, + "inactive_split_bytes_peak_delta": 1075248128, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 12880707584, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 12880707584, + "segment_count_current": 1815, + "segment_count_current_delta": -197, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185038957530Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.009790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185038957530Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185038957530Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f5cacd9114c1f7e9377402adca5c6ad754f6d8a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185038957530Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c59e05ff20a42c991e55e5d8364a3308d658f5fecec233a6d5a22507a484ce9b +size 2771660 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185039331880Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185039331880Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81d152d49130ec8cfc74d9f57b8d841d6fcff6aa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185039331880Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 6, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.331490Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 6, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.015438Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12988620288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185039331880Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.341797Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185039331880Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185039331880Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0c1ef4bb216b1ddca6f867399395a0053fc0f224 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185039331880Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ac67631e8c7e21b399d771649b59dc7c7d24fc48f2155bdd39e7a8af42c3f2 +size 416220 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185039653087Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185039653087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8dea3d257c796fb33f19cfd4b9f7e9af53a58995 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185039653087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 6, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.652700Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 6, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 12988620288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.342913Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12988620288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185039653087Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.662787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185039653087Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185039653087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b25049b69631a31df1c99472eae75b87fa41f903 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185039653087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7b621a2148dcbcc41b1dff3d9b76c3977c722f4b08ca5d3c31c59fee2576102 +size 416220 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185034928303Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185034928303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c96395a6d5aa919551dddf7a561d2e983ba5bc3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185034928303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 40774371840, + "active_bytes_peak": 90612776448, + "active_bytes_peak_delta": 46067000832, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 7, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.927816Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 40774371840, + "allocated_bytes_peak": 90612776448, + "allocated_bytes_peak_delta": 46067000832, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 7, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:49:54.861496Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_current_delta": 1145599488, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 10888034816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 103244890112, + "reserved_bytes_current_delta": 54993616896, + "reserved_bytes_peak": 103244890112, + "reserved_bytes_peak_delta": 54993616896, + "segment_count_current": 1790, + "segment_count_current_delta": 1169, + "segment_count_peak": 1790, + "segment_count_peak_delta": 1169, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185034928303Z.pickle", + "timestamp_utc": "2026-08-26T18:50:34.980742Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185034928303Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185034928303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..677bd021e381e2207334014c5219bfec9d5b8960 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185034928303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c0da58ab93ff36fe75181c4c61fc5d3bfb406db4ee17ac26ef7b2144814ebcf +size 3148002 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185035546628Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185035546628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54c2c23a1cecd6b072583909404117ea30c6a06f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185035546628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478210048, + "active_bytes_peak_delta": 30865433600, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 7, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.546207Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478210048, + "allocated_bytes_peak_delta": 30865433600, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 90612776448, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 90612776448, + "device": 7, + "inactive_split_bytes_current": 1566957056, + "inactive_split_bytes_peak": 11913372160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 103244890112, + "reserved_bytes_peak": 103244890112, + "segment_count_current": 1790, + "segment_count_peak": 1790, + "timestamp_utc": "2026-08-26T18:50:34.987117Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 25560088576, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 25560088576, + "segment_count_current": 2012, + "segment_count_current_delta": 222, + "segment_count_peak": 2012, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185035546628Z.pickle", + "timestamp_utc": "2026-08-26T18:50:35.596737Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185035546628Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185035546628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b7cefa189c23b3e5b6ad258ba221f80532e95d5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185035546628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e4bd859ce6badeaa82542d7a05ba9d9fe1cf627444d744473c84b799ad3f1c2 +size 2579980 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185035972080Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185035972080Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c890fe1558123f0b7335fae4fc99a862ad3b503c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185035972080Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294981632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478455808, + "active_bytes_peak_delta": 245760, + "after": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 7, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.971662Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294981632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121478455808, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478210048, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478210048, + "device": 7, + "inactive_split_bytes_current": 1969610240, + "inactive_split_bytes_peak": 11913372160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:35.603336Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11913372160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 128804978688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128804978688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2012, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185035972080Z.pickle", + "timestamp_utc": "2026-08-26T18:50:36.017791Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185035972080Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185035972080Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b94896da249d1b93da018384a6c7ade3fa063c2e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185035972080Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf78223fa1f4841c12db6fd672343bc892b5590b9839609972dc40c0c7ab6a5d +size 2569036 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185038957452Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185038957452Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e73b8f92c36002b838051fd83d6be716618726e7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185038957452Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 7256715776, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 7, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:38.956991Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 8958464, + "before": { + "active_bytes_current": 85294981632, + "active_bytes_peak": 121478455808, + "allocated_bytes_current": 85294981632, + "allocated_bytes_peak": 121478455808, + "device": 7, + "inactive_split_bytes_current": 1971707392, + "inactive_split_bytes_peak": 11913372160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 128804978688, + "reserved_bytes_peak": 128804978688, + "segment_count_current": 2012, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:36.023156Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": -319033344, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 1170605568, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 12880707584, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 12880707584, + "segment_count_current": 1815, + "segment_count_current_delta": -197, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185038957452Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.011418Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185038957452Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185038957452Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b08529d33688512356fdc7ff5dc797acfebde469 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185038957452Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:296adc53bbc988d2af1476e76478c6b29a4ff4a0095bbe3d097ce86ff25e3bde +size 2771680 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185039331846Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185039331846Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24c365cbc2c97db5c94cf2f2ce8416a2b1152505 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185039331846Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 7, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.331445Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 7, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.018627Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185039331846Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.341371Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185039331846Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185039331846Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d282fe52f0fc4931c56e8aa7c8b9bad0629c2a39 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185039331846Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1cd18905e65f74466dacb2369bf0fe4f5bfcf5306c3dbd654c6c08df256af93 +size 416153 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185039653036Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185039653036Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc2fd1ff703844661ea6ee88b93769576c8b5ede --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185039653036Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312025088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128735171584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 7, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.652659Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312025088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121487414272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312025088, + "active_bytes_peak": 128735171584, + "allocated_bytes_current": 85312025088, + "allocated_bytes_peak": 121487414272, + "device": 7, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_peak": 13083977728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 141685686272, + "reserved_bytes_peak": 141685686272, + "segment_count_current": 1815, + "segment_count_peak": 2012, + "timestamp_utc": "2026-08-26T18:50:39.342483Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1652674048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13083977728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 141685686272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141685686272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1815, + "segment_count_current_delta": 0, + "segment_count_peak": 2012, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185039653036Z.pickle", + "timestamp_utc": "2026-08-26T18:50:39.666444Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185039653036Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185039653036Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c96281dd742284b3fb21fdace0dde08db0a11396 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185039653036Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67bf597867ea0ef15749029e56b607076c37c86c32ccf793a6e144e2bf23a786 +size 416153 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log index fcd145d6dc03f3e3b1180f02e905db0502c57162..df80be36339224a4b7a85df3476e60e5d7105413 100644 --- a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -1265,3 +1265,115 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 18:50:34] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 40065.6 | throughput per GPU (TFLOP/s/GPU): 4.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116019E+01 | loss scale: 1.0 | grad norm: 44.780 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.85 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81341.69970703125 | max allocated: 86413.26904296875 | reserved: 98394.0 | max reserved: 98394.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 18:50:35] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 618.8 | throughput per GPU (TFLOP/s/GPU): 292.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117609E+01 | loss scale: 1.0 | grad norm: 44.316 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:50:35] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 425.6 | throughput per GPU (TFLOP/s/GPU): 424.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.694874E+00 | loss scale: 1.0 | grad norm: 66.665 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 18:50:38] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2985.2 | throughput per GPU (TFLOP/s/GPU): 60.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338633E+01 | loss scale: 1.0 | grad norm: 19.896 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:50:39] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 374.5 | throughput per GPU (TFLOP/s/GPU): 482.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.003646E+01 | loss scale: 1.0 | grad norm: 81.280 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:50:39] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 321.2 | throughput per GPU (TFLOP/s/GPU): 563.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.411165E+01 | loss scale: 1.0 | grad norm: 121.472 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:50:39 +[rank0]:[W826 18:50:40.860399159 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..5605a4185531a41eb4a5fe98ce6e640d048fdf8a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 4361 +1, 4313 +2, 4697 +3, 4673 +4, 4337 +5, 4313 +6, 4625 +7, 4745 +0, 48169 +1, 48241 +2, 48153 +3, 48151 +4, 47881 +5, 48187 +6, 48147 +7, 48179 +0, 49929 +1, 49929 +2, 49929 +3, 49925 +4, 49641 +5, 49927 +6, 49929 +7, 49929 +0, 53699 +1, 53699 +2, 53699 +3, 53699 +4, 53411 +5, 53699 +6, 53699 +7, 53699 +0, 54107 +1, 54107 +2, 54107 +3, 54107 +4, 53819 +5, 54107 +6, 54107 +7, 54107 +0, 54437 +1, 54443 +2, 54435 +3, 54447 +4, 54147 +5, 54415 +6, 54451 +7, 54451 +0, 55545 +1, 55545 +2, 55545 +3, 55545 +4, 55257 +5, 55545 +6, 55545 +7, 55545 +0, 56121 +1, 56121 +2, 56121 +3, 56121 +4, 55833 +5, 56121 +6, 56121 +7, 56121 +0, 57147 +1, 57147 +2, 57147 +3, 57147 +4, 56859 +5, 57147 +6, 57147 +7, 57147 +0, 57845 +1, 57843 +2, 57843 +3, 57843 +4, 57557 +5, 57843 +6, 57843 +7, 57843 +0, 97389 +1, 97397 +2, 97397 +3, 97397 +4, 97469 +5, 97397 +6, 97461 +7, 97461 +0, 98101 +1, 98109 +2, 98109 +3, 98109 +4, 98117 +5, 98109 +6, 98109 +7, 98109 +0, 98101 +1, 98109 +2, 98109 +3, 98109 +4, 98117 +5, 98109 +6, 98109 +7, 98109 +0, 98101 +1, 98109 +2, 98109 +3, 98109 +4, 98117 +5, 98109 +6, 98109 +7, 98109 +0, 98101 +1, 98109 +2, 98109 +3, 98109 +4, 98117 +5, 98109 +6, 98109 +7, 98109 +0, 98101 +1, 98109 +2, 98109 +3, 98109 +4, 98117 +5, 98109 +6, 98109 +7, 98109 +0, 99467 +1, 99475 +2, 99475 +3, 99475 +4, 99483 +5, 99475 +6, 99475 +7, 99475 +0, 103187 +1, 103195 +2, 103195 +3, 103195 +4, 103203 +5, 103195 +6, 103195 +7, 103195 +0, 107029 +1, 107149 +2, 107165 +3, 107199 +4, 107211 +5, 107133 +6, 107163 +7, 107139 +0, 121477 +1, 120755 +2, 116447 +3, 121391 +4, 93471 +5, 122031 +6, 111263 +7, 119567 +0, 146189 +1, 146245 +2, 146245 +3, 146245 +4, 146517 +5, 146245 +6, 146245 +7, 146245 +0, 145589 +1, 145319 +2, 145011 +3, 145209 +4, 145395 +5, 145645 +6, 145319 +7, 145063 +0, 144941 +1, 4646 +2, 3366 +3, 3366 +4, 3366 +5, 3366 +6, 3366 +7, 3622 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185155262312Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185155262312Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a29e8ea6dbec0871ec2973b920ca2947b6eaeb6a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185155262312Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294068224, + "active_bytes_current_delta": 40773458432, + "active_bytes_peak": 91778436608, + "active_bytes_peak_delta": 47232660992, + "after": { + "active_bytes_current": 85294068224, + "active_bytes_peak": 91778436608, + "allocated_bytes_current": 85294068224, + "allocated_bytes_peak": 91778436608, + "device": 0, + "inactive_split_bytes_current": 1301532160, + "inactive_split_bytes_peak": 12995810816, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 101802049536, + "reserved_bytes_peak": 101802049536, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.261843Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294068224, + "allocated_bytes_current_delta": 40773458432, + "allocated_bytes_peak": 91778436608, + "allocated_bytes_peak_delta": 47232660992, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 0, + "inactive_split_bytes_current": 446523392, + "inactive_split_bytes_peak": 1050503168, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:51:15.489592Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1301532160, + "inactive_split_bytes_current_delta": 855008768, + "inactive_split_bytes_peak": 12995810816, + "inactive_split_bytes_peak_delta": 11945307648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 101802049536, + "reserved_bytes_current_delta": 53550776320, + "reserved_bytes_peak": 101802049536, + "reserved_bytes_peak_delta": 53550776320, + "segment_count_current": 1621, + "segment_count_current_delta": 1000, + "segment_count_peak": 1621, + "segment_count_peak_delta": 1000, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185155262312Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.316994Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185155262312Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185155262312Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c673aab674b0ea764f30f8503de52b254d7f530 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185155262312Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6352e00a813317a1582faee73c3794eec94b023d6cb5af660b38a8b9dff1c681 +size 3138979 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185155933780Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185155933780Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a067ee05a58a8b04f9be91708d1032de6febd006 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185155933780Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294068224, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122655462400, + "active_bytes_peak_delta": 30877025792, + "after": { + "active_bytes_current": 85294068224, + "active_bytes_peak": 122655462400, + "allocated_bytes_current": 85294068224, + "allocated_bytes_peak": 122655462400, + "device": 0, + "inactive_split_bytes_current": 1706282496, + "inactive_split_bytes_peak": 12995810816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 130027618304, + "reserved_bytes_peak": 130027618304, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:55.933376Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294068224, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122655462400, + "allocated_bytes_peak_delta": 30877025792, + "before": { + "active_bytes_current": 85294068224, + "active_bytes_peak": 91778436608, + "allocated_bytes_current": 85294068224, + "allocated_bytes_peak": 91778436608, + "device": 0, + "inactive_split_bytes_current": 1301532160, + "inactive_split_bytes_peak": 12995810816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 101802049536, + "reserved_bytes_peak": 101802049536, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.323392Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1706282496, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 12995810816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 130027618304, + "reserved_bytes_current_delta": 28225568768, + "reserved_bytes_peak": 130027618304, + "reserved_bytes_peak_delta": 28225568768, + "segment_count_current": 1923, + "segment_count_current_delta": 302, + "segment_count_peak": 1923, + "segment_count_peak_delta": 302, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185155933780Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.981592Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185155933780Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185155933780Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf9b550af201747bae7120caa4322b4aeda2e73b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185155933780Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:358e485d1081c61f6933728eb54b0fb3f73e9817a8c444434d0865252b2122ae +size 2657763 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185156359485Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185156359485Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd255ccb02b6c49b2b369b511c822810982eaa57 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185156359485Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85294068224, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122657575936, + "active_bytes_peak_delta": 2113536, + "after": { + "active_bytes_current": 85294068224, + "active_bytes_peak": 122657575936, + "allocated_bytes_current": 85294068224, + "allocated_bytes_peak": 122657575936, + "device": 0, + "inactive_split_bytes_current": 1706282496, + "inactive_split_bytes_peak": 12995810816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 130027618304, + "reserved_bytes_peak": 130027618304, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:56.359076Z", + "world_size": 8 + }, + "allocated_bytes_current": 85294068224, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122657575936, + "allocated_bytes_peak_delta": 2113536, + "before": { + "active_bytes_current": 85294068224, + "active_bytes_peak": 122655462400, + "allocated_bytes_current": 85294068224, + "allocated_bytes_peak": 122655462400, + "device": 0, + "inactive_split_bytes_current": 1706282496, + "inactive_split_bytes_peak": 12995810816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 130027618304, + "reserved_bytes_peak": 130027618304, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:55.987839Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1706282496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12995810816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 130027618304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130027618304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1923, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185156359485Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.404573Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185156359485Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185156359485Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06fc8672578afe8e3e366dd97d37f6257c53c785 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185156359485Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b38d418507f799c05b0f60f3a79cdd91b96757820b9a2a3e551731aa8f1d696 +size 2646047 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185159255047Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185159255047Z.json new file mode 100644 index 0000000000000000000000000000000000000000..799c48a11c5c8811a8a942138e63e38dae0e8743 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185159255047Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85311111680, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129901028352, + "active_bytes_peak_delta": 7243452416, + "after": { + "active_bytes_current": 85311111680, + "active_bytes_peak": 129901028352, + "allocated_bytes_current": 85311111680, + "allocated_bytes_peak": 122657575936, + "device": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_peak": 12995810816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_peak": 142723776512, + "segment_count_current": 1875, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:59.254500Z", + "world_size": 8 + }, + "allocated_bytes_current": 85311111680, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122657575936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85294068224, + "active_bytes_peak": 122657575936, + "allocated_bytes_current": 85294068224, + "allocated_bytes_peak": 122657575936, + "device": 0, + "inactive_split_bytes_current": 1706282496, + "inactive_split_bytes_peak": 12995810816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 130027618304, + "reserved_bytes_peak": 130027618304, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:56.409882Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_current_delta": -191107072, + "inactive_split_bytes_peak": 12995810816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_current_delta": 12696158208, + "reserved_bytes_peak": 142723776512, + "reserved_bytes_peak_delta": 12696158208, + "segment_count_current": 1875, + "segment_count_current_delta": -48, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185159255047Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.309555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185159255047Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185159255047Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e86aba93c7a5cd05900f70e7b7c54813b408c549 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185159255047Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22f7765cc1a9ea713b5b0daa72915f28e49a8a9909fba7543a3acdad8127c833 +size 3520796 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185159631759Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185159631759Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d2439666905a321c2db616c8fd575085f4cd562 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185159631759Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85311111680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129901028352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85311111680, + "active_bytes_peak": 129901028352, + "allocated_bytes_current": 85311111680, + "allocated_bytes_peak": 122657575936, + "device": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_peak": 12995810816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_peak": 142723776512, + "segment_count_current": 1875, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:59.631362Z", + "world_size": 8 + }, + "allocated_bytes_current": 85311111680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122657575936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85311111680, + "active_bytes_peak": 129901028352, + "allocated_bytes_current": 85311111680, + "allocated_bytes_peak": 122657575936, + "device": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_peak": 12995810816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_peak": 142723776512, + "segment_count_current": 1875, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:59.315517Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12995810816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142723776512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1875, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185159631759Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.642045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185159631759Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185159631759Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67c46359c31dbe577274f55e4485413a4dd7e2bf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185159631759Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:631a1bf52a4210e63958d08404a75059622a2a883ba7e4994576cefc9ffd7a0f +size 561954 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185159957148Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185159957148Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d92c15ec653ab98f43f0481a70ab6d9cd88fd0e6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185159957148Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85311111680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129901028352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85311111680, + "active_bytes_peak": 129901028352, + "allocated_bytes_current": 85311111680, + "allocated_bytes_peak": 122657575936, + "device": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_peak": 12995810816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_peak": 142723776512, + "segment_count_current": 1875, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:59.956764Z", + "world_size": 8 + }, + "allocated_bytes_current": 85311111680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122657575936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85311111680, + "active_bytes_peak": 129901028352, + "allocated_bytes_current": 85311111680, + "allocated_bytes_peak": 122657575936, + "device": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_peak": 12995810816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_peak": 142723776512, + "segment_count_current": 1875, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T18:51:59.643172Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1515175424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12995810816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142723776512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142723776512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1875, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185159957148Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.967125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185159957148Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185159957148Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e4343d4dbab86b0c2ecc77091f6f3e977a627de --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185159957148Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0705fbe69f3b05ba926cc7740990694398d5cc232695e79679442dd6de40fac6 +size 561954 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185155262305Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185155262305Z.json new file mode 100644 index 0000000000000000000000000000000000000000..333a4104f13c77b34da8f0ba8a880a4abf582612 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185155262305Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 40772902912, + "active_bytes_peak": 91782173696, + "active_bytes_peak_delta": 47236398080, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 1, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.261786Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 40772902912, + "allocated_bytes_peak": 91782173696, + "allocated_bytes_peak_delta": 47236398080, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 1, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:51:15.489660Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_current_delta": 941547520, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 11864860672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 101936267264, + "reserved_bytes_current_delta": 53684994048, + "reserved_bytes_peak": 101936267264, + "reserved_bytes_peak_delta": 53684994048, + "segment_count_current": 1621, + "segment_count_current_delta": 1000, + "segment_count_peak": 1621, + "segment_count_peak_delta": 1000, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185155262305Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.313532Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185155262305Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185155262305Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0abd2e40538c19e89317d5bf23e019fdcfe16a55 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185155262305Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeaf8f2916d01914f0688a092c84d392ec81da5e7ea1734f11a8d2b131419588 +size 3122373 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185155933707Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185155933707Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f104ee3b3cdab27d859d02a318b3ac5351b70de --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185155933707Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122650515968, + "active_bytes_peak_delta": 30868342272, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 1, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.933280Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122650515968, + "allocated_bytes_peak_delta": 30868342272, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 1, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.320572Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 28198305792, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 28198305792, + "segment_count_current": 1920, + "segment_count_current_delta": 299, + "segment_count_peak": 1920, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185155933707Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.985023Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185155933707Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185155933707Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c0a324da55ad8800e914c26c27588dc9d2d5ee85 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185155933707Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c330ff3a6a6a84415b618e25a6e78565c9dbc756db9029d86d2c7cc8bb99961 +size 2572190 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185156359440Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185156359440Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35ae7426fc952e3e47c2a1e648edcb96fb94db4a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185156359440Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122652613120, + "active_bytes_peak_delta": 2097152, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122652613120, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 1, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.359013Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652613120, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 1, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.991017Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1920, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185156359440Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.404946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185156359440Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185156359440Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b56fe0b96f2a2fc63fa8953999293d4ef1dee00d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185156359440Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33a4c00734b554640ade3ffba126dac13f0beb73426f59a4a7032807178f6570 +size 2580632 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185159254966Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185159254966Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f60eba6dc04ce8d6d7f27f2e1435b37e12c73080 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185159254966Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 7247859712, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 1, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.254438Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 102400, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122652613120, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 1, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.410333Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": -191107072, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 12647923712, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 12647923712, + "segment_count_current": 1873, + "segment_count_current_delta": -47, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185159254966Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.309938Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185159254966Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185159254966Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..271a5a43ac160502d9f94283af1c954f889aa4e2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185159254966Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3db189b630b40abf5d253b156827ddb3ddce782a2e94fd81c98a5154d12df0e +size 3244596 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185159631669Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185159631669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66725f03595eae0c75b167fc729bae931a790626 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185159631669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 1, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.631279Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 1, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.315881Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185159631669Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.641757Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185159631669Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185159631669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3dfaebe9ef80655a5f6dd43884330107ba0df45 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185159631669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07386e020fa5386d6771578d4f1e8bf7d523a369a222d1f87897c118fbe3e032 +size 516637 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185159957290Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185159957290Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be77708f2c14ecc61e09043e14855673be0f166f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185159957290Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 1, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.956768Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 1, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.642904Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185159957290Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.967859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185159957290Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185159957290Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef4563935df75b9672a8ce32076b70802c5f9dc5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185159957290Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac328aa1253d695c8f0bf2d34ea92ffe5885c76a5fcfefba65b7240782de3a65 +size 516606 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185155262292Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185155262292Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf8e81c2422db3c264e16f6ceaa8a117399f23de --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185155262292Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 40772902912, + "active_bytes_peak": 91782173696, + "active_bytes_peak_delta": 47236398080, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 2, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.261758Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 40772902912, + "allocated_bytes_peak": 91782173696, + "allocated_bytes_peak_delta": 47236398080, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 2, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:51:15.489592Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_current_delta": 941547520, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 11864860672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 101936267264, + "reserved_bytes_current_delta": 53684994048, + "reserved_bytes_peak": 101936267264, + "reserved_bytes_peak_delta": 53684994048, + "segment_count_current": 1621, + "segment_count_current_delta": 1000, + "segment_count_peak": 1621, + "segment_count_peak_delta": 1000, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185155262292Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.314050Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185155262292Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185155262292Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..198d9375af08430f011fae25895b6ea1b3feefc0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185155262292Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feae2d16a874d7fa74286bc2669df6fb2a514566af7546308c5ad6bbeb2a08bd +size 3122370 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185155933657Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185155933657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40f9b7ee429acfbe91a5ebe337bcd38b117fa46e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185155933657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122650515968, + "active_bytes_peak_delta": 30868342272, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 2, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.933261Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122650515968, + "allocated_bytes_peak_delta": 30868342272, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 2, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.320207Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 28198305792, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 28198305792, + "segment_count_current": 1920, + "segment_count_current_delta": 299, + "segment_count_peak": 1920, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185155933657Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.980930Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185155933657Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185155933657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1e1c01dbf228bc6a98b08b69934d36500aec420 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185155933657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6a0765a07cbc86363056e7f13bb19fc25c66c4b982b65badf0c9d4816d661d9 +size 2576209 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185156359411Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185156359411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07afea96af1c54459850b9512ae503b365a7bc62 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185156359411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123256592896, + "active_bytes_peak_delta": 606076928, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 2, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.359014Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652613120, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 2, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.986301Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1920, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185156359411Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.402443Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185156359411Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185156359411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b1d3291a0ad51920b877ab86f5ae62f6b86590d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185156359411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cae8ac7b5e180c51d890a4596416728f4b5d95812cc13680d12de58081b245b +size 2580428 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185159254967Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185159254967Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f41a8c77f897f46c6019b35945b749cc6676f8f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185159254967Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 6643879936, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 2, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.254431Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 102400, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 2, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.407367Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": -191107072, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 12647923712, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 12647923712, + "segment_count_current": 1873, + "segment_count_current_delta": -47, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185159254967Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.307563Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185159254967Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185159254967Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02659e6b03fbcb81c75d729ebd3af181d28b3783 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185159254967Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb596821eba7c898c9711ff0956f9fa8ff8ae7bf14d77bdaf82c6fe0bb0396ca +size 3244240 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185159631669Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185159631669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be2e2094993b5b3c119ab162ce0b929f7c64b314 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185159631669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 2, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.631272Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 2, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.313219Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185159631669Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.641592Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185159631669Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185159631669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d4420d6d2da9b5ee8297ac6345e21f3107a629a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185159631669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b5da38ece4d72947fdf46f1d590cc7ad9d2bd2834ee8c007dd3afa27db7341c +size 516773 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185159957075Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185159957075Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8db1de32ee1498f1d0a63b8a4af882a69fda259 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185159957075Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 2, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.956687Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 2, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.642723Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185159957075Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.966634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185159957075Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185159957075Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1aca87c8f51c256af7c064f74025f53cf31bb2d1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185159957075Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d63c95ec101206f4a31607d10d100040a734a13b599690c48619b955d56078bd +size 516742 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185155262303Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185155262303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e78ad25ec1c311b52db43f03ed2681213e082e4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185155262303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 40772902912, + "active_bytes_peak": 91782173696, + "active_bytes_peak_delta": 47236398080, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 3, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.261771Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 40772902912, + "allocated_bytes_peak": 91782173696, + "allocated_bytes_peak_delta": 47236398080, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 3, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:51:15.489647Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_current_delta": 941547520, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 11864860672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 101936267264, + "reserved_bytes_current_delta": 53684994048, + "reserved_bytes_peak": 101936267264, + "reserved_bytes_peak_delta": 53684994048, + "segment_count_current": 1621, + "segment_count_current_delta": 1000, + "segment_count_peak": 1621, + "segment_count_peak_delta": 1000, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185155262303Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.313690Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185155262303Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185155262303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e2064e738ec085e3fe53114d3151a1eb1ee30af --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185155262303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ee53e68610a2e0c9f7efb453872a90cfa69c70c2c8f400e01e9cfa728d85b04 +size 3122373 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185155933655Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185155933655Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b1f728a37a6a5dab64361001851e3aebce3ebed --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185155933655Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122650515968, + "active_bytes_peak_delta": 30868342272, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 3, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.933241Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122650515968, + "allocated_bytes_peak_delta": 30868342272, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 3, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.320019Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 28198305792, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 28198305792, + "segment_count_current": 1920, + "segment_count_current_delta": 299, + "segment_count_peak": 1920, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185155933655Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.981894Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185155933655Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185155933655Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..853b996b628a6667788eba9d0797a704f401b919 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185155933655Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e75cc51d337de57cb9e9c4af90774958ef248a983284238880d1a6485c3dc96f +size 2576209 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185156359392Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185156359392Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be7045c2a03abee00194889f432a081948b842c9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185156359392Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123256592896, + "active_bytes_peak_delta": 606076928, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 3, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.358983Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652613120, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 3, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.987290Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1920, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185156359392Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.402842Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185156359392Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185156359392Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..428bde3215f7e4b840ca25c92a72ea5e19ac1e91 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185156359392Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437c94fe98f6877e78b82f756cf2580494409f955b7fbcb81ddff052c9403a20 +size 2580445 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185159254986Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185159254986Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5130c30b598b6da086ae69afe12a462bc75907ea --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185159254986Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 6643879936, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 3, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.254431Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 102400, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 3, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.407831Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": -191107072, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 12647923712, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 12647923712, + "segment_count_current": 1873, + "segment_count_current_delta": -47, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185159254986Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.308003Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185159254986Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185159254986Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..426300bd0c54f51859a01d8d0bb81e95b88c0334 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185159254986Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34f629fdddd788d44b410884e30bc5e21c50640604460c54586980c83a57f51e +size 3244248 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185159631660Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185159631660Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c37af32058d8bf250d1849192393705fb1623bbc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185159631660Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 3, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.631269Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 3, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.313925Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185159631660Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.641480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185159631660Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185159631660Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c178cd3f1957551f5c63685614ce5a4fc5aa7183 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185159631660Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d20e8366b1c059f1e7fff4b3db3c34a2b559ebd7d1888a4e950020e728ad8c0c +size 516781 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185159957074Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185159957074Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab66f0a86d2a430df8466a73dece1faac03a3d8d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185159957074Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 3, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.956698Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 3, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.642602Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185159957074Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.966854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185159957074Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185159957074Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dc082c3d819c02aa5b8df9b40f56a48688d54c0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185159957074Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62088e510263dea2669263e5ccc043afb0457d0efe65e4cb6ff1d77a1e70e143 +size 516750 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185155262384Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185155262384Z.json new file mode 100644 index 0000000000000000000000000000000000000000..387d5dbd44d8a0a5a38d332380096bd5ea0ddb57 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185155262384Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297115136, + "active_bytes_current_delta": 40776505344, + "active_bytes_peak": 91796119040, + "active_bytes_peak_delta": 47250343424, + "after": { + "active_bytes_current": 85297115136, + "active_bytes_peak": 91796119040, + "allocated_bytes_current": 85297115136, + "allocated_bytes_peak": 91796119040, + "device": 4, + "inactive_split_bytes_current": 1640321024, + "inactive_split_bytes_peak": 12758440960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 101969821696, + "reserved_bytes_peak": 101969821696, + "segment_count_current": 1628, + "segment_count_peak": 1628, + "timestamp_utc": "2026-08-26T18:51:55.261839Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297115136, + "allocated_bytes_current_delta": 40776505344, + "allocated_bytes_peak": 91796119040, + "allocated_bytes_peak_delta": 47250343424, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 4, + "inactive_split_bytes_current": 270362624, + "inactive_split_bytes_peak": 874342400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T18:51:15.489565Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1640321024, + "inactive_split_bytes_current_delta": 1369958400, + "inactive_split_bytes_peak": 12758440960, + "inactive_split_bytes_peak_delta": 11884098560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 101969821696, + "reserved_bytes_current_delta": 54020538368, + "reserved_bytes_peak": 101969821696, + "reserved_bytes_peak_delta": 54020538368, + "segment_count_current": 1628, + "segment_count_current_delta": 1013, + "segment_count_peak": 1628, + "segment_count_peak_delta": 1013, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185155262384Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.315493Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185155262384Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185155262384Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6c3604dd29546121a2d385082e30b0a52c38e61 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185155262384Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20abaaa6ac2b995dd742bdf0be61549a83f99984fb545a63d01b1bd6557d13f4 +size 3130733 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185155933667Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185155933667Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9dc386fe1c078ce01f0de5be7047b965bf586859 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185155933667Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297115136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122673035776, + "active_bytes_peak_delta": 30876916736, + "after": { + "active_bytes_current": 85297115136, + "active_bytes_peak": 122673035776, + "allocated_bytes_current": 85297115136, + "allocated_bytes_peak": 122673035776, + "device": 4, + "inactive_split_bytes_current": 2045071360, + "inactive_split_bytes_peak": 12758440960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 130121990144, + "reserved_bytes_peak": 130121990144, + "segment_count_current": 1924, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:55.933268Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297115136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122673035776, + "allocated_bytes_peak_delta": 30876916736, + "before": { + "active_bytes_current": 85297115136, + "active_bytes_peak": 91796119040, + "allocated_bytes_current": 85297115136, + "allocated_bytes_peak": 91796119040, + "device": 4, + "inactive_split_bytes_current": 1640321024, + "inactive_split_bytes_peak": 12758440960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 101969821696, + "reserved_bytes_peak": 101969821696, + "segment_count_current": 1628, + "segment_count_peak": 1628, + "timestamp_utc": "2026-08-26T18:51:55.321675Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2045071360, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 12758440960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 130121990144, + "reserved_bytes_current_delta": 28152168448, + "reserved_bytes_peak": 130121990144, + "reserved_bytes_peak_delta": 28152168448, + "segment_count_current": 1924, + "segment_count_current_delta": 296, + "segment_count_peak": 1924, + "segment_count_peak_delta": 296, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185155933667Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.980521Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185155933667Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185155933667Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..04012da9e7f6f0fcbc72bf0faf3b350ae42ce4f5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185155933667Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:341bc0d42f5001d300e4269136f97184a277b9e65db9f904024bf548cfab6041 +size 2694731 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185156359401Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185156359401Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79ef5f7bd392c5d1c4887bc66d6e9d7d289cabc5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185156359401Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85297115136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123284336128, + "active_bytes_peak_delta": 611300352, + "after": { + "active_bytes_current": 85297115136, + "active_bytes_peak": 123284336128, + "allocated_bytes_current": 85297115136, + "allocated_bytes_peak": 122680356352, + "device": 4, + "inactive_split_bytes_current": 2047168512, + "inactive_split_bytes_peak": 12758440960, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 130121990144, + "reserved_bytes_peak": 130121990144, + "segment_count_current": 1924, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:56.359004Z", + "world_size": 8 + }, + "allocated_bytes_current": 85297115136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122680356352, + "allocated_bytes_peak_delta": 7320576, + "before": { + "active_bytes_current": 85297115136, + "active_bytes_peak": 122673035776, + "allocated_bytes_current": 85297115136, + "allocated_bytes_peak": 122673035776, + "device": 4, + "inactive_split_bytes_current": 2045071360, + "inactive_split_bytes_peak": 12758440960, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 130121990144, + "reserved_bytes_peak": 130121990144, + "segment_count_current": 1924, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:55.985571Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2047168512, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12758440960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 130121990144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130121990144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1924, + "segment_count_current_delta": 0, + "segment_count_peak": 1924, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185156359401Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.402006Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185156359401Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185156359401Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de353a7a84daa0614f50f16339836d9449d8ca43 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185156359401Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ac05d98839b7c84f04a4b162fb25b5b87847dd1eed792c499fa1a53418c2a3 +size 2639674 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185159254979Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185159254979Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6760794c165a036368cc6971379f9a528de22525 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185159254979Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85314158592, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129904075264, + "active_bytes_peak_delta": 6619739136, + "after": { + "active_bytes_current": 85314158592, + "active_bytes_peak": 129904075264, + "allocated_bytes_current": 85314158592, + "allocated_bytes_peak": 122680356352, + "device": 4, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_peak": 12758440960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 143067709440, + "reserved_bytes_peak": 143067709440, + "segment_count_current": 1902, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:59.254441Z", + "world_size": 8 + }, + "allocated_bytes_current": 85314158592, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122680356352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85297115136, + "active_bytes_peak": 123284336128, + "allocated_bytes_current": 85297115136, + "allocated_bytes_peak": 122680356352, + "device": 4, + "inactive_split_bytes_current": 2047168512, + "inactive_split_bytes_peak": 12758440960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 130121990144, + "reserved_bytes_peak": 130121990144, + "segment_count_current": 1924, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:56.406645Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_current_delta": -193204224, + "inactive_split_bytes_peak": 12758440960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 143067709440, + "reserved_bytes_current_delta": 12945719296, + "reserved_bytes_peak": 143067709440, + "reserved_bytes_peak_delta": 12945719296, + "segment_count_current": 1902, + "segment_count_current_delta": -22, + "segment_count_peak": 1924, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185159254979Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.307625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185159254979Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185159254979Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ab8c337aa5b5969816448695313094ae8cb290c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185159254979Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff19278c1d68ff572bca2b35be4f18cfaf4960556842040abe77863effda8e7 +size 2748775 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185159631680Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185159631680Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40a1a587f6a3999907c49e5333fe9ba37351d992 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185159631680Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85314158592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129904075264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85314158592, + "active_bytes_peak": 129904075264, + "allocated_bytes_current": 85314158592, + "allocated_bytes_peak": 122680356352, + "device": 4, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_peak": 12758440960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 143067709440, + "reserved_bytes_peak": 143067709440, + "segment_count_current": 1902, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:59.631298Z", + "world_size": 8 + }, + "allocated_bytes_current": 85314158592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122680356352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85314158592, + "active_bytes_peak": 129904075264, + "allocated_bytes_current": 85314158592, + "allocated_bytes_peak": 122680356352, + "device": 4, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_peak": 12758440960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 143067709440, + "reserved_bytes_peak": 143067709440, + "segment_count_current": 1902, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:59.313074Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12758440960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 143067709440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143067709440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1902, + "segment_count_current_delta": 0, + "segment_count_peak": 1924, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185159631680Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.641513Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185159631680Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185159631680Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c2bdc6ccd293d4b5b5a8babf13fc187de6536ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185159631680Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7ff04576e765c909636ebe848ff364bd07d90bae2ec9d2efe4b638fb8f42ac3 +size 410391 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185159957072Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185159957072Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e96bbec06804f2d12c09965d401db5f795f09849 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185159957072Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85314158592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129904075264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85314158592, + "active_bytes_peak": 129904075264, + "allocated_bytes_current": 85314158592, + "allocated_bytes_peak": 122680356352, + "device": 4, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_peak": 12758440960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 143067709440, + "reserved_bytes_peak": 143067709440, + "segment_count_current": 1902, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:59.956698Z", + "world_size": 8 + }, + "allocated_bytes_current": 85314158592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122680356352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85314158592, + "active_bytes_peak": 129904075264, + "allocated_bytes_current": 85314158592, + "allocated_bytes_peak": 122680356352, + "device": 4, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_peak": 12758440960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 143067709440, + "reserved_bytes_peak": 143067709440, + "segment_count_current": 1902, + "segment_count_peak": 1924, + "timestamp_utc": "2026-08-26T18:51:59.642644Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1853964288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12758440960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 143067709440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143067709440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1902, + "segment_count_current_delta": 0, + "segment_count_peak": 1924, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185159957072Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.966685Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185159957072Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185159957072Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da51816a166c6af322fea5bef15ad28dea9097eb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185159957072Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b5e80958ad0e0d6c17c495acb9dcf5a1dc47990a982dbd39801c816ab3b01b +size 410391 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185155262305Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185155262305Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38ab4f86e348d8d1be57faef7f5877d208763c05 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185155262305Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 40772902912, + "active_bytes_peak": 91782173696, + "active_bytes_peak_delta": 47236398080, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 5, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.261790Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 40772902912, + "allocated_bytes_peak": 91782173696, + "allocated_bytes_peak_delta": 47236398080, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 5, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:51:15.489634Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_current_delta": 941547520, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 11864860672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 101936267264, + "reserved_bytes_current_delta": 53684994048, + "reserved_bytes_peak": 101936267264, + "reserved_bytes_peak_delta": 53684994048, + "segment_count_current": 1621, + "segment_count_current_delta": 1000, + "segment_count_peak": 1621, + "segment_count_peak_delta": 1000, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185155262305Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.316274Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185155262305Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185155262305Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73bbf503a63d45c57d7b7718ea32a0d0367eca5b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185155262305Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fecbc1f77aef69814d528de49911a12c434a50c9ab5a8f4bca79b933e8f7386 +size 3122373 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185155933650Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185155933650Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9fb35164e940905c8dfc7d3dcb7b5599faa7299c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185155933650Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122650515968, + "active_bytes_peak_delta": 30868342272, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 5, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.933240Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122650515968, + "allocated_bytes_peak_delta": 30868342272, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 5, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.322208Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 28198305792, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 28198305792, + "segment_count_current": 1920, + "segment_count_current_delta": 299, + "segment_count_peak": 1920, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185155933650Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.983439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185155933650Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185155933650Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d44d6be5ce89f7a9312b1d8fa2f157bc2bf5367 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185155933650Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03972938c5dcd2c34c9f0a4190e8dc52b4207e58e3d4c18180d40e921e5b6379 +size 2572151 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185156359392Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185156359392Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57c67d56b00180eefd259739340e8730308dbcb7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185156359392Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122652613120, + "active_bytes_peak_delta": 2097152, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122652613120, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 5, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.358972Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652613120, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 5, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.989216Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1920, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185156359392Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.405148Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185156359392Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185156359392Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a03e629772ed93c5d7b3301907d2d91f1001491 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185156359392Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c640b168f0cd2407f44a750dc9f1909eb08e27ddb743112793c4b647132b202 +size 2580589 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185159254966Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185159254966Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80f17b6c4bcc5ede4dfcb729c15ed85472d87265 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185159254966Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 7247859712, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 5, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.254414Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 102400, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122652613120, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 5, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.410412Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": -191107072, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 12647923712, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 12647923712, + "segment_count_current": 1873, + "segment_count_current_delta": -47, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185159254966Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.308157Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185159254966Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185159254966Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bdc388fdbc86c8344f61192b405da792b7abdee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185159254966Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98ec60cb355bc634a81cdb9e1bd0a8f2e957089275407efc106c6dbab6fa04c7 +size 3244426 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185159631633Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185159631633Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4a196207330da9688a1d9b2789a089715c6fc2e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185159631633Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 5, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.631240Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 5, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.314022Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185159631633Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.641637Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185159631633Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185159631633Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cf775b42c635934cccb0b983f358b17216b4038 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185159631633Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2e7b7132a2eaa7bd67543beacac83b226fbbe55ec2d34e64f5cd937c86badae +size 516637 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185159957049Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185159957049Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8faad96988f80ddb6816b8e4bb22e50f3eba3c9d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185159957049Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 5, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.956660Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 5, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.642773Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185159957049Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.966849Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185159957049Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185159957049Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b8a836da48156fe114ab16c27340e9ecaaf2cd92 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185159957049Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0805bceaf90654e5c316ad7cf62759a4978c7926b8f28e4d7d0f4527c2d4262f +size 516606 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185155262309Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185155262309Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1d0bd733d793ec6ad3ea5b4eacbc983f57b4930 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185155262309Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 40772902912, + "active_bytes_peak": 91782173696, + "active_bytes_peak_delta": 47236398080, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 6, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.261814Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 40772902912, + "allocated_bytes_peak": 91782173696, + "allocated_bytes_peak_delta": 47236398080, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 6, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:51:15.489617Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_current_delta": 941547520, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 11864860672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 101936267264, + "reserved_bytes_current_delta": 53684994048, + "reserved_bytes_peak": 101936267264, + "reserved_bytes_peak_delta": 53684994048, + "segment_count_current": 1621, + "segment_count_current_delta": 1000, + "segment_count_peak": 1621, + "segment_count_peak_delta": 1000, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185155262309Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.317196Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185155262309Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185155262309Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95bed171865b3347fbede16772171af4a0ca05fd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185155262309Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cada3f55e475b1ffc2ce93544d1709d339a5034e5571bd30c228f5eefd274d +size 3122373 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185155933685Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185155933685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6fd8d751bf3a9df48cbaf7d040e2d1f437416d01 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185155933685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122650515968, + "active_bytes_peak_delta": 30868342272, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 6, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.933274Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122650515968, + "allocated_bytes_peak_delta": 30868342272, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 6, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.323572Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 28198305792, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 28198305792, + "segment_count_current": 1920, + "segment_count_current_delta": 299, + "segment_count_peak": 1920, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185155933685Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.981339Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185155933685Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185155933685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b183e1d51ee717f12c5b7bc3860323b2a815161c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185155933685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20eed1dd16d4498f6e5c1728fd6d68a683c4c311b1d79fc6c9aa05c1b3cd364 +size 2572151 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185156359377Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185156359377Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bd2a293ddf351675348e9d58a5163a34b02c09f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185156359377Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123256592896, + "active_bytes_peak_delta": 606076928, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 6, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.358980Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652613120, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 6, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.986645Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1920, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185156359377Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.402945Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185156359377Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185156359377Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0491b259d1fb19d9ee17b4b79ffcd947db5f771d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185156359377Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01955adb6e6119f46396e52845ca2f8eaa9045f52a75ba39d92a5c246ffbed99 +size 2580481 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185159254950Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185159254950Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99f02e994238799a2a913b749d5215057bb0a536 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185159254950Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 6643879936, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 6, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.254409Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 102400, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 6, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.407809Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": -191107072, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 12647923712, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 12647923712, + "segment_count_current": 1873, + "segment_count_current_delta": -47, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185159254950Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.308690Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185159254950Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185159254950Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8232a13fb6eab7444da3667dec228d0e75947933 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185159254950Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc2a0eddcec410f4cd7f81fc7c2ffd908d1e2c2035ed44f3f762d7cde8c88e10 +size 3244252 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185159631725Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185159631725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1c9addc17a7f95704096894f9d919c926fb953c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185159631725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 6, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.631333Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 6, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.314407Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185159631725Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.641826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185159631725Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185159631725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..edf7e6135f6bb8ba8113507c1c8ad8ae33551d35 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185159631725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb6279f11b94d2825ddcd0704c4212e58726bc9510239a4954335bdd2fadf47f +size 516635 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185159957088Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185159957088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a8e75ced641130c1644112d295aaae4c788112c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185159957088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 6, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.956703Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 6, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.642972Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185159957088Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.966844Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185159957088Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185159957088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b19f3ecd9af78a8b17f3af0d5534e65c6dfebc0c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185159957088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2fe122f9151d48794a56af77b29fc0b14e1165d88e5e6a7aebe1e54edcb3480 +size 516604 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185155262490Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185155262490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c2d6ce6dee3c7fc5203e699c511fb83a8bd1eda --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185155262490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 40772902912, + "active_bytes_peak": 91782173696, + "active_bytes_peak_delta": 47236398080, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 7, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.261956Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 40772902912, + "allocated_bytes_peak": 91782173696, + "allocated_bytes_peak_delta": 47236398080, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 7, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:51:15.489600Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_current_delta": 941547520, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 11864860672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 101936267264, + "reserved_bytes_current_delta": 53684994048, + "reserved_bytes_peak": 101936267264, + "reserved_bytes_peak_delta": 53684994048, + "segment_count_current": 1621, + "segment_count_current_delta": 1000, + "segment_count_peak": 1621, + "segment_count_peak_delta": 1000, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185155262490Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.316355Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185155262490Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185155262490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..594ddb46a5632fdab241f9713438e55cc5f250ca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185155262490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a954ec236bf791ff741aaa95d4870ed006145c108c5a2cacba0f14ab9910010 +size 3122373 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185155933679Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185155933679Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59ff88b2884d8f3277847ac5457ef45b5c8fe5a1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185155933679Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122650515968, + "active_bytes_peak_delta": 30868342272, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 7, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.933279Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122650515968, + "allocated_bytes_peak_delta": 30868342272, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 91782173696, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 91782173696, + "device": 7, + "inactive_split_bytes_current": 1362905088, + "inactive_split_bytes_peak": 12890198016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 101936267264, + "reserved_bytes_peak": 101936267264, + "segment_count_current": 1621, + "segment_count_peak": 1621, + "timestamp_utc": "2026-08-26T18:51:55.322641Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 28198305792, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 28198305792, + "segment_count_current": 1920, + "segment_count_current_delta": 299, + "segment_count_peak": 1920, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185155933679Z.pickle", + "timestamp_utc": "2026-08-26T18:51:55.981217Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185155933679Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185155933679Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..32ed237c9f9c41b71330ac45696647851d22f74d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185155933679Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33f3ce1c22eb4c73570d17b8097c413ffae296a090697f66545f650b74e149db +size 2572149 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185156359440Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185156359440Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc273407b6b8ae7c55b2b3335d8277df36f62ba0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185156359440Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85293512704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123256592896, + "active_bytes_peak_delta": 606076928, + "after": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 7, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.359040Z", + "world_size": 8 + }, + "allocated_bytes_current": 85293512704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652613120, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 122650515968, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122650515968, + "device": 7, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:55.986413Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 130134573056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130134573056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1920, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185156359440Z.pickle", + "timestamp_utc": "2026-08-26T18:51:56.402618Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185156359440Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185156359440Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2694a6f4b9129d1fe26bd775c5fa73b4ccc9f6ae --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185156359440Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4605ae10d32c39bc59dbfbf7fb6dd0dd019f6caa30439f6d069e210ece0606 +size 2580600 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185159254949Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185159254949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43a72db8d40f05aeac2e75751d1d45b90d0991b1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185159254949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 6643879936, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 7, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.254431Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 102400, + "before": { + "active_bytes_current": 85293512704, + "active_bytes_peak": 123256592896, + "allocated_bytes_current": 85293512704, + "allocated_bytes_peak": 122652613120, + "device": 7, + "inactive_split_bytes_current": 1765558272, + "inactive_split_bytes_peak": 12890198016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 130134573056, + "reserved_bytes_peak": 130134573056, + "segment_count_current": 1920, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:56.407325Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": -191107072, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 12647923712, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 12647923712, + "segment_count_current": 1873, + "segment_count_current_delta": -47, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185159254949Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.308312Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185159254949Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185159254949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9faac4acf1563a0f4c1150d677fc468da725dec8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185159254949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97f2d11f0b0851d163a66670085aa1ac31ed385e1c3f3c85645944c4106229dc +size 3244444 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185159631698Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185159631698Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fcda3aa54a370bdbd122ca704fb8c8364835b7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185159631698Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 7, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.631309Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 7, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.313901Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185159631698Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.641669Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185159631698Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185159631698Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03df06b804d632f77c0d3c340cfa37d5c9d43567 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185159631698Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4766aeb70eababbe764bd3fb41d68a6a266b0f05363705cf8f26eec4add795ae +size 516635 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185159957088Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185159957088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b66d06be0e738643f77c8dfb9a2e6dead5b88f0e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185159957088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85310556160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129900472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 7, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.956712Z", + "world_size": 8 + }, + "allocated_bytes_current": 85310556160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122652715520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85310556160, + "active_bytes_peak": 129900472832, + "allocated_bytes_current": 85310556160, + "allocated_bytes_peak": 122652715520, + "device": 7, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_peak": 12890198016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142782496768, + "reserved_bytes_peak": 142782496768, + "segment_count_current": 1873, + "segment_count_peak": 1920, + "timestamp_utc": "2026-08-26T18:51:59.642793Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1574451200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12890198016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142782496768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142782496768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1873, + "segment_count_current_delta": 0, + "segment_count_peak": 1920, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185159957088Z.pickle", + "timestamp_utc": "2026-08-26T18:51:59.966727Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185159957088Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185159957088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc05dc88034ff4355fd4676fff15f685fc0f5133 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185159957088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7388b32c50179bdf4eb0e2fe30bca9a9ff71404cfdad7dabae28e0dbf3a8f0da +size 516604 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..e08c27d6064f18975ee6629a93e270875bd5322e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.125 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..731c9e8f2061b248e14dc0d4ca69118ea47d44cb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log @@ -0,0 +1,1381 @@ +[18:50:48] WARN: better-memory-dump-support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.125 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 18:50:50.276000 566471 torch/distributed/run.py:766] +W0826 18:50:50.276000 566471 torch/distributed/run.py:766] ***************************************** +W0826 18:50:50.276000 566471 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:50:50.276000 566471 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7[Gloo] Rank peer ranks. 7Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. [Gloo] Rank 0Expected number of connected peer ranks is : peer ranks. 00Expected number of connected peer ranks is : is connected to +00 + peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W826 18:51:08.913010797 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:51:08.913073740 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:51:08.913096554 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:51:08.913108893 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:51:08.913152588 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:51:08.913699356 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:51:08.915350620 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.032 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 18:51:09.099780902 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.702 seconds +time to initialize megatron (seconds): 7.073 +[after megatron is initialized] datetime: 2026-08-26 18:51:14 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:51:15 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 18:51:15 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (625.87, 650.25) + train/valid/test-data-iterators-setup ..........: (57.66, 66.38) +[before the start of training step] datetime: 2026-08-26 18:51:15 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +Number of parameters in transformer block in billions: 29.85 [2026-08-26 18:51:55] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 39771.4 | throughput per GPU (TFLOP/s/GPU): 4.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.777 | number of skipped iterations: 0 | number of nan iterations: 0 | + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81342.76220703125 | max allocated: 87526.73779296875 | reserved: 97086.0 | max reserved: 97086.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 18:51:55] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 672.2 | throughput per GPU (TFLOP/s/GPU): 269.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.305 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:51:56] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 425.7 | throughput per GPU (TFLOP/s/GPU): 424.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.694060E+00 | loss scale: 1.0 | grad norm: 66.446 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 18:51:59] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2895.3 | throughput per GPU (TFLOP/s/GPU): 62.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338558E+01 | loss scale: 1.0 | grad norm: 19.939 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:51:59] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 376.9 | throughput per GPU (TFLOP/s/GPU): 479.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.008962E+01 | loss scale: 1.0 | grad norm: 83.289 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:51:59] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 325.4 | throughput per GPU (TFLOP/s/GPU): 555.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.364389E+01 | loss scale: 1.0 | grad norm: 121.709 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:51:59 +[rank0]:[W826 18:52:01.340514497 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..ee922c169a37f35720802cf8cc836dadf197fe50 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,248 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 47 +2, 629 +3, 629 +4, 629 +5, 47 +6, 629 +7, 47 +0, 939 +1, 937 +2, 927 +3, 951 +4, 955 +5, 949 +6, 933 +7, 969 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 47803 +1, 47803 +2, 47803 +3, 47803 +4, 47515 +5, 47803 +6, 47803 +7, 47803 +0, 49281 +1, 49057 +2, 49281 +3, 49281 +4, 48993 +5, 49057 +6, 49057 +7, 49281 +0, 49929 +1, 49929 +2, 49929 +3, 49929 +4, 49641 +5, 49929 +6, 49931 +7, 49929 +0, 53699 +1, 53699 +2, 53699 +3, 53699 +4, 53411 +5, 53699 +6, 53699 +7, 53699 +0, 54107 +1, 54107 +2, 54107 +3, 54107 +4, 53819 +5, 54107 +6, 54107 +7, 54107 +0, 54369 +1, 54397 +2, 54385 +3, 54397 +4, 54083 +5, 54385 +6, 54419 +7, 54399 +0, 55545 +1, 55545 +2, 55545 +3, 55545 +4, 55257 +5, 55545 +6, 55545 +7, 55545 +0, 56185 +1, 56185 +2, 56185 +3, 56185 +4, 55897 +5, 56185 +6, 56185 +7, 56185 +0, 57323 +1, 57323 +2, 57323 +3, 57323 +4, 57035 +5, 57323 +6, 57323 +7, 57323 +0, 58097 +1, 58095 +2, 58095 +3, 58095 +4, 57809 +5, 58095 +6, 58095 +7, 58095 +0, 98913 +1, 98969 +2, 98981 +3, 98965 +4, 98925 +5, 98971 +6, 98977 +7, 98977 +0, 99365 +1, 99407 +2, 99407 +3, 99407 +4, 99377 +5, 99407 +6, 99407 +7, 99407 +0, 99365 +1, 99407 +2, 99407 +3, 99407 +4, 99377 +5, 99407 +6, 99407 +7, 99407 +0, 99365 +1, 99407 +2, 99407 +3, 99407 +4, 99377 +5, 99407 +6, 99407 +7, 99407 +0, 99365 +1, 99407 +2, 99407 +3, 99407 +4, 99377 +5, 99407 +6, 99407 +7, 99407 +0, 100731 +1, 100773 +2, 100773 +3, 100773 +4, 100743 +5, 100773 +6, 100773 +7, 100773 +0, 104451 +1, 104493 +2, 104493 +3, 104493 +4, 104463 +5, 104493 +6, 104493 +7, 104493 +0, 108669 +1, 108749 +2, 108691 +3, 108761 +4, 108823 +5, 108695 +6, 108731 +7, 108725 +0, 127545 +1, 127375 +2, 127495 +3, 127531 +4, 127107 +5, 127411 +6, 125707 +7, 127663 +0, 147771 +1, 147719 +2, 147931 +3, 147911 +4, 148163 +5, 147911 +6, 147911 +7, 147931 +0, 147769 +1, 147717 +2, 5886 +3, 147909 +4, 148161 +5, 147909 +6, 147909 +7, 147929 +0, 3963 +1, 3965 +2, 3966 +3, 4286 +4, 148161 +5, 5566 +6, 3966 +7, 3966 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185313625646Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185313625646Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8da28538062f2de243a39eaf83459c0bb8d80163 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185313625646Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85292937728, + "active_bytes_current_delta": 40772327936, + "active_bytes_peak": 92951842304, + "active_bytes_peak_delta": 48406066688, + "after": { + "active_bytes_current": 85292937728, + "active_bytes_peak": 92951842304, + "allocated_bytes_current": 85292937728, + "allocated_bytes_peak": 92951842304, + "device": 0, + "inactive_split_bytes_current": 1340411392, + "inactive_split_bytes_peak": 11124289024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 103530102784, + "reserved_bytes_peak": 103530102784, + "segment_count_current": 1760, + "segment_count_peak": 1760, + "timestamp_utc": "2026-08-26T18:53:13.624977Z", + "world_size": 8 + }, + "allocated_bytes_current": 85292937728, + "allocated_bytes_current_delta": 40772327936, + "allocated_bytes_peak": 92951842304, + "allocated_bytes_peak_delta": 48406066688, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 0, + "inactive_split_bytes_current": 446523392, + "inactive_split_bytes_peak": 1050503168, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:52:35.069341Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1340411392, + "inactive_split_bytes_current_delta": 893888000, + "inactive_split_bytes_peak": 11124289024, + "inactive_split_bytes_peak_delta": 10073785856, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 103530102784, + "reserved_bytes_current_delta": 55278829568, + "reserved_bytes_peak": 103530102784, + "reserved_bytes_peak_delta": 55278829568, + "segment_count_current": 1760, + "segment_count_current_delta": 1139, + "segment_count_peak": 1760, + "segment_count_peak_delta": 1139, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185313625646Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.681221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185313625646Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185313625646Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ae7502e6c1b95d3faea92cd40102241125cb867 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185313625646Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666ab9291ef345cc1921ab698ee05fe1985830c99bfe1a46cb7984a17d7944d7 +size 3151374 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185314224433Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185314224433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b35d18fb0d115a9a58964167ed12395d226fe1a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185314224433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85292937728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123818988544, + "active_bytes_peak_delta": 30867146240, + "after": { + "active_bytes_current": 85292937728, + "active_bytes_peak": 123818988544, + "allocated_bytes_current": 85292937728, + "allocated_bytes_peak": 123818988544, + "device": 0, + "inactive_split_bytes_current": 1747258880, + "inactive_split_bytes_peak": 11124289024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 131386572800, + "reserved_bytes_peak": 131386572800, + "segment_count_current": 1936, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:14.224007Z", + "world_size": 8 + }, + "allocated_bytes_current": 85292937728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123818988544, + "allocated_bytes_peak_delta": 30867146240, + "before": { + "active_bytes_current": 85292937728, + "active_bytes_peak": 92951842304, + "allocated_bytes_current": 85292937728, + "allocated_bytes_peak": 92951842304, + "device": 0, + "inactive_split_bytes_current": 1340411392, + "inactive_split_bytes_peak": 11124289024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 103530102784, + "reserved_bytes_peak": 103530102784, + "segment_count_current": 1760, + "segment_count_peak": 1760, + "timestamp_utc": "2026-08-26T18:53:13.687919Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1747258880, + "inactive_split_bytes_current_delta": 406847488, + "inactive_split_bytes_peak": 11124289024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 131386572800, + "reserved_bytes_current_delta": 27856470016, + "reserved_bytes_peak": 131386572800, + "reserved_bytes_peak_delta": 27856470016, + "segment_count_current": 1936, + "segment_count_current_delta": 176, + "segment_count_peak": 1936, + "segment_count_peak_delta": 176, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185314224433Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.560155Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185314224433Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185314224433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..490bc877c36c2879fb99980fd010fabda26ff11c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185314224433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc70ada57fe3048f4d645ca84d050de51b79f3aa268204cc4153d4937aa363f9 +size 2582688 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185314987719Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185314987719Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ec66e9bc57ba819942129bbef9941957f7a43300 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185314987719Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85292937728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125630927872, + "active_bytes_peak_delta": 1811939328, + "after": { + "active_bytes_current": 85292937728, + "active_bytes_peak": 125630927872, + "allocated_bytes_current": 85292937728, + "allocated_bytes_peak": 123818988544, + "device": 0, + "inactive_split_bytes_current": 1946488320, + "inactive_split_bytes_peak": 11124289024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 131386572800, + "reserved_bytes_peak": 131386572800, + "segment_count_current": 1936, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:14.987322Z", + "world_size": 8 + }, + "allocated_bytes_current": 85292937728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123818988544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85292937728, + "active_bytes_peak": 123818988544, + "allocated_bytes_current": 85292937728, + "allocated_bytes_peak": 123818988544, + "device": 0, + "inactive_split_bytes_current": 1747258880, + "inactive_split_bytes_peak": 11124289024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 131386572800, + "reserved_bytes_peak": 131386572800, + "segment_count_current": 1936, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:14.569145Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1946488320, + "inactive_split_bytes_current_delta": 199229440, + "inactive_split_bytes_peak": 11124289024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 131386572800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131386572800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1936, + "segment_count_current_delta": 0, + "segment_count_peak": 1936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185314987719Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.036596Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185314987719Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185314987719Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf295bc1d2ae582280fe5966819460f32548e817 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185314987719Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23f69d49766b31ff7dc1dba98b0863e6b5795b9c1d3784eacdeecbfe8bffc395 +size 2618911 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185317851720Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185317851720Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c84d5b06d4ed8a7f4ab310d8a3303f5df3b1ce1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185317851720Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85309981184, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131074204672, + "active_bytes_peak_delta": 5443276800, + "after": { + "active_bytes_current": 85309981184, + "active_bytes_peak": 131074204672, + "allocated_bytes_current": 85309981184, + "allocated_bytes_peak": 123826447360, + "device": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_peak": 12561481216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_peak": 144382623744, + "segment_count_current": 1793, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:17.851143Z", + "world_size": 8 + }, + "allocated_bytes_current": 85309981184, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123826447360, + "allocated_bytes_peak_delta": 7458816, + "before": { + "active_bytes_current": 85292937728, + "active_bytes_peak": 125630927872, + "allocated_bytes_current": 85292937728, + "allocated_bytes_peak": 123818988544, + "device": 0, + "inactive_split_bytes_current": 1946488320, + "inactive_split_bytes_peak": 11124289024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 131386572800, + "reserved_bytes_peak": 131386572800, + "segment_count_current": 1936, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:15.042645Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_current_delta": -597954560, + "inactive_split_bytes_peak": 12561481216, + "inactive_split_bytes_peak_delta": 1437192192, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_current_delta": 12996050944, + "reserved_bytes_peak": 144382623744, + "reserved_bytes_peak_delta": 12996050944, + "segment_count_current": 1793, + "segment_count_current_delta": -143, + "segment_count_peak": 1936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185317851720Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.907742Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185317851720Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185317851720Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..440a71bf789c97abacb8c795379e84fd0ce17dff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185317851720Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:725fbb3d3fbea630e3f5629f2d3718000bae2a5f2116307800b609c66d2efba4 +size 3388822 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185318232520Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185318232520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50c1afdb5d97c06229563f4eaad03aaa1fb4d9ce --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185318232520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85309981184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131074204672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85309981184, + "active_bytes_peak": 131074204672, + "allocated_bytes_current": 85309981184, + "allocated_bytes_peak": 123826447360, + "device": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_peak": 12561481216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_peak": 144382623744, + "segment_count_current": 1793, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:18.232123Z", + "world_size": 8 + }, + "allocated_bytes_current": 85309981184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123826447360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85309981184, + "active_bytes_peak": 131074204672, + "allocated_bytes_current": 85309981184, + "allocated_bytes_peak": 123826447360, + "device": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_peak": 12561481216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_peak": 144382623744, + "segment_count_current": 1793, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:17.914313Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12561481216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144382623744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1793, + "segment_count_current_delta": 0, + "segment_count_peak": 1936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185318232520Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.243693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185318232520Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185318232520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b236e7f45c7dcf2199bab9795b09a76fa8aa20de --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185318232520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00d56a61d1c9dfa96f6d11b40228b8efef339e89381b7e3d928c21d56077f6ab +size 519501 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185318559262Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185318559262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b54c2ee03d45998b256942e02075e22752014df1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185318559262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85309981184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131074204672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85309981184, + "active_bytes_peak": 131074204672, + "allocated_bytes_current": 85309981184, + "allocated_bytes_peak": 123826447360, + "device": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_peak": 12561481216, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_peak": 144382623744, + "segment_count_current": 1793, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:18.558881Z", + "world_size": 8 + }, + "allocated_bytes_current": 85309981184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123826447360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85309981184, + "active_bytes_peak": 131074204672, + "allocated_bytes_current": 85309981184, + "allocated_bytes_peak": 123826447360, + "device": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_peak": 12561481216, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_peak": 144382623744, + "segment_count_current": 1793, + "segment_count_peak": 1936, + "timestamp_utc": "2026-08-26T18:53:18.244835Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1348533760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12561481216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 144382623744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144382623744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1793, + "segment_count_current_delta": 0, + "segment_count_peak": 1936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185318559262Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.569494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185318559262Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185318559262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db31f256d4906771595cf132c7dc7fb20f7d9757 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185318559262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8eb467cb5440acb145e2fc9916935bf493048a8048289ad55822fcd4300ce32 +size 519501 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185313625790Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185313625790Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36518f50d813962469bc6ae3d2a6e16912ff8400 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185313625790Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 40775332352, + "active_bytes_peak": 92952749568, + "active_bytes_peak_delta": 48406973952, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 1, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.625130Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 40775332352, + "allocated_bytes_peak": 92952749568, + "allocated_bytes_peak_delta": 48406973952, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 1, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:52:35.069605Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_current_delta": 1064947200, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 10830632448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 103574142976, + "reserved_bytes_current_delta": 55322869760, + "reserved_bytes_peak": 103574142976, + "reserved_bytes_peak_delta": 55322869760, + "segment_count_current": 1763, + "segment_count_current_delta": 1142, + "segment_count_peak": 1763, + "segment_count_peak_delta": 1142, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185313625790Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.678976Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185313625790Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185313625790Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f44672a6f3bce71e2c1ad78eefe734e7395a123d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185313625790Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73d7eea774837f3f1e7276adac86cd7dbd6145a7decca945f88781c187abd703 +size 3134747 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185314224424Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185314224424Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c825189ae810b9164e4d1af34d120172eaf308be --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185314224424Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123820125184, + "active_bytes_peak_delta": 30867375616, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 1, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.224001Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820125184, + "allocated_bytes_peak_delta": 30867375616, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 1, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.685551Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 27948744704, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 27948744704, + "segment_count_current": 1946, + "segment_count_current_delta": 183, + "segment_count_peak": 1946, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185314224424Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.608601Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185314224424Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185314224424Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77355309bd00b87e50a91bcb232c338c09df0758 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185314224424Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf73a02330c7db9e4a00d18162f01fefaa29312b784293229adc0ede89329f81 +size 2577429 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185314987771Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185314987771Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b780624150a88e3ad47cbd034771f084d63406b8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185314987771Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123820370944, + "active_bytes_peak_delta": 245760, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820370944, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 1, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.987380Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820370944, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 1, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.616947Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1946, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185314987771Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.035165Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185314987771Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185314987771Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d56830617769a5cc45743ffebcb3b030654a06e5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185314987771Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edddbddc9dff832da3585ab5dea7822d9ab2974318eb2c7c0dddde2d89b399e3 +size 2598857 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185317851600Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185317851600Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5d39cd2f1e6f412ca6e1ce1aca8e8a3ab613f25 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185317851600Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 7256838144, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 1, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12521527296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 144328097792, + "reserved_bytes_peak": 144328097792, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.851131Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 9080832, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820370944, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 1, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:15.041151Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": -396627968, + "inactive_split_bytes_peak": 12521527296, + "inactive_split_bytes_peak_delta": 665557504, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 144328097792, + "reserved_bytes_current_delta": 12805210112, + "reserved_bytes_peak": 144328097792, + "reserved_bytes_peak_delta": 12805210112, + "segment_count_current": 1800, + "segment_count_current_delta": -146, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185317851600Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.907508Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185317851600Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185317851600Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..997fc9de05892b7776d73a6240f76f365820416a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185317851600Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a6f59a45ccf1d801212208faf5201d0a35901224307bcb108d5ae6ea857ac8d +size 3223558 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185318232535Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185318232535Z.json new file mode 100644 index 0000000000000000000000000000000000000000..77e9542193b76488624019f5721ac03e191fd7c8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185318232535Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 1, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12521527296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 144328097792, + "reserved_bytes_peak": 144328097792, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.232130Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 1, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12521527296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 144328097792, + "reserved_bytes_peak": 144328097792, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.914108Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12521527296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 144328097792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144328097792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185318232535Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.242892Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185318232535Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185318232535Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..561d16efd683e238f2d1e7f785e4d295c23bc677 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185318232535Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e5801148d21d6413b5200f6f3c1397044d842204d4efc60c1e38a0636508cc6 +size 491040 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185318559271Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185318559271Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2bc8ed877ebfdf9b1d3e0683b9688ff1cda12511 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185318559271Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 1, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12521527296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 144328097792, + "reserved_bytes_peak": 144328097792, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.558885Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 1, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12521527296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 144328097792, + "reserved_bytes_peak": 144328097792, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.244012Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12521527296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 144328097792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144328097792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185318559271Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.569129Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185318559271Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185318559271Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a36b446f7f4793aff4fbf7608d94daaf61f27a45 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185318559271Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aab8abcc00e5f2252b84fc038ef78f7f57b06621cceb9efeebe1d8f0887ca14 +size 491040 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185313625647Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185313625647Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16cca3034addfc12c65895fbf1c900669a8fb474 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185313625647Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 40775332352, + "active_bytes_peak": 92952749568, + "active_bytes_peak_delta": 48406973952, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 2, + "inactive_split_bytes_current": 1507276288, + "inactive_split_bytes_peak": 11876941312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.624971Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 40775332352, + "allocated_bytes_peak": 92952749568, + "allocated_bytes_peak_delta": 48406973952, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 2, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:52:35.069342Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1507276288, + "inactive_split_bytes_current_delta": 1085918720, + "inactive_split_bytes_peak": 11876941312, + "inactive_split_bytes_peak_delta": 10851603968, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 103574142976, + "reserved_bytes_current_delta": 55322869760, + "reserved_bytes_peak": 103574142976, + "reserved_bytes_peak_delta": 55322869760, + "segment_count_current": 1763, + "segment_count_current_delta": 1142, + "segment_count_peak": 1763, + "segment_count_peak_delta": 1142, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185313625647Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.678702Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185313625647Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185313625647Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe6a2cdfa0cb3b4e22b2f6a2ed2a0de0f0664a7d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185313625647Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:122311c9fae5fdb8d98c2ba3ded2815fd1f40f8dfbe14593f3ff429f72a144c7 +size 3134754 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185314224408Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185314224408Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5e1d2c41b454ed9e44c8d1b6700ffae86cd61db --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185314224408Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123820370944, + "active_bytes_peak_delta": 30867621376, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820370944, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 2, + "inactive_split_bytes_current": 1909929472, + "inactive_split_bytes_peak": 11876941312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.223984Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820370944, + "allocated_bytes_peak_delta": 30867621376, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 2, + "inactive_split_bytes_current": 1507276288, + "inactive_split_bytes_peak": 11876941312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.685205Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1909929472, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11876941312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 27948744704, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 27948744704, + "segment_count_current": 1946, + "segment_count_current_delta": 183, + "segment_count_peak": 1946, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185314224408Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.543494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185314224408Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185314224408Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..539b145eff96ab1eecef3eeda4ae10be9d38c92d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185314224408Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f14499126560d17a568245bf7a57f7a8a25dfd4f7c70924cec1440d26ac94ea4 +size 2573447 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185314987715Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185314987715Z.json new file mode 100644 index 0000000000000000000000000000000000000000..660e52e7aba645255f474b48582ba6a93c64f431 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185314987715Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125632556032, + "active_bytes_peak_delta": 1812185088, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632556032, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820616704, + "device": 2, + "inactive_split_bytes_current": 2111256064, + "inactive_split_bytes_peak": 11876941312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.987304Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820616704, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820370944, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 2, + "inactive_split_bytes_current": 1909929472, + "inactive_split_bytes_peak": 11876941312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.551471Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2111256064, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 11876941312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1946, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185314987715Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.035492Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185314987715Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185314987715Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db9e80f8e618a9bd45f0b1762d2f950ec0dd4ba4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185314987715Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26fa86e0049732d782e4c6c16233e7ece9c51e5f2ab68e3a47e72615be7ba2cf +size 2585026 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185317851581Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185317851581Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a89606df1338fc963d9c05d9b878a6e079776255 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185317851581Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 5444653056, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 2, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.851106Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 8835072, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632556032, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820616704, + "device": 2, + "inactive_split_bytes_current": 2111256064, + "inactive_split_bytes_peak": 11876941312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:15.041433Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_current_delta": -597954560, + "inactive_split_bytes_peak": 12743825408, + "inactive_split_bytes_peak_delta": 866884096, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 144550395904, + "reserved_bytes_current_delta": 13027508224, + "reserved_bytes_peak": 144550395904, + "reserved_bytes_peak_delta": 13027508224, + "segment_count_current": 1801, + "segment_count_current_delta": -145, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185317851581Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.905666Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185317851581Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185317851581Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6dc1f529c3353043af2e161f204b0f22c66b8e60 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185317851581Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:705112e64a90fe8f8f39ed7da06542322714082745f428343dc1bf85ea001dd2 +size 3223764 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185318232460Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185318232460Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27cc4dedbb5ba3c75d7e323783b63f20135be529 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185318232460Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 2, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.232068Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 2, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.911941Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12743825408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 144550395904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144550395904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1801, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185318232460Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.242778Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185318232460Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185318232460Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6f994799b6f975163ea3d5cb7deffaa01c68294 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185318232460Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14338b6e295f6b9b6ceee5caa8b4bbd6216b77bac9b6e8ffad2f5c715cbb07e6 +size 490857 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185318559259Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185318559259Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27e28a6efe247f924ddd8e41611b4fc84976bcd7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185318559259Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 2, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.558868Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 2, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_peak": 12743825408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 144550395904, + "reserved_bytes_peak": 144550395904, + "segment_count_current": 1801, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.243936Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1513301504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12743825408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 144550395904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144550395904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1801, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185318559259Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.568771Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185318559259Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185318559259Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53fa2a1e1201b63d14eaa9b74c10f7d803b25be3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185318559259Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0c7aed5e5f835229a0078dc93c4ddaf7260817b3c0dd2d49f4a38d84ec7da12 +size 490857 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185313625648Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185313625648Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7a5c858fccaf7ba981ec86222afaed428d04d25 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185313625648Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 40775332352, + "active_bytes_peak": 92952749568, + "active_bytes_peak_delta": 48406973952, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 3, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.624973Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 40775332352, + "allocated_bytes_peak": 92952749568, + "allocated_bytes_peak_delta": 48406973952, + "before": { + "active_bytes_current": 44520609792, + "active_bytes_peak": 44545775616, + "allocated_bytes_current": 44520609792, + "allocated_bytes_peak": 44545775616, + "device": 3, + "inactive_split_bytes_current": 421357568, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T18:52:35.069363Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_current_delta": 1064947200, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 10830632448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 103574142976, + "reserved_bytes_current_delta": 55322869760, + "reserved_bytes_peak": 103574142976, + "reserved_bytes_peak_delta": 55322869760, + "segment_count_current": 1763, + "segment_count_current_delta": 1142, + "segment_count_peak": 1763, + "segment_count_peak_delta": 1142, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185313625648Z.pickle", + "timestamp_utc": "2026-08-26T18:53:13.677460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185313625648Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185313625648Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ce464a38aedb714419df89b9a45301b82c46958 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185313625648Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16174c3e9f44f8890e70fa61a221f607f9fc321a74ae4436408f4533133a3ff2 +size 3134747 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185314224492Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185314224492Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7985f7a7f2b54fcf20f246be4a9f1091d1a19628 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185314224492Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123820125184, + "active_bytes_peak_delta": 30867375616, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 3, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.224079Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820125184, + "allocated_bytes_peak_delta": 30867375616, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 92952749568, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 92952749568, + "device": 3, + "inactive_split_bytes_current": 1486304768, + "inactive_split_bytes_peak": 11855969792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 103574142976, + "reserved_bytes_peak": 103574142976, + "segment_count_current": 1763, + "segment_count_peak": 1763, + "timestamp_utc": "2026-08-26T18:53:13.683768Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_current_delta": 402653184, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 27948744704, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 27948744704, + "segment_count_current": 1946, + "segment_count_current_delta": 183, + "segment_count_peak": 1946, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185314224492Z.pickle", + "timestamp_utc": "2026-08-26T18:53:14.542859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185314224492Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185314224492Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3b9505a451ee548da202834b6c54af69448d0d31 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185314224492Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39786a0ecebd626112470899725b36bec56803d9d32dec71f40c8bae9ec2a3d +size 2573411 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185314987725Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185314987725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0d00e42b95e831e01d220b27d9e5a4e433e89c9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185314987725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85295942144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125632310272, + "active_bytes_peak_delta": 1812185088, + "after": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632310272, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 3, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.987336Z", + "world_size": 8 + }, + "allocated_bytes_current": 85295942144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123820370944, + "allocated_bytes_peak_delta": 245760, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 123820125184, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820125184, + "device": 3, + "inactive_split_bytes_current": 1888957952, + "inactive_split_bytes_peak": 11855969792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:14.550830Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 11855969792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 131522887680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131522887680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1946, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185314987725Z.pickle", + "timestamp_utc": "2026-08-26T18:53:15.033293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185314987725Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185314987725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5df1334e64dbff592cf3d293785fc4a4c680ec18 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185314987725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f6e2b27872cf389cddd94a373165231ba4b23acf136e1fddbf08c91b8579e6 +size 2585069 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185317851553Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185317851553Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d23d435d26e2b6d419c08bc825e2c7416175b9c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185317851553Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 5444898816, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 3, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.851105Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 9080832, + "before": { + "active_bytes_current": 85295942144, + "active_bytes_peak": 125632310272, + "allocated_bytes_current": 85295942144, + "allocated_bytes_peak": 123820370944, + "device": 3, + "inactive_split_bytes_current": 2090284544, + "inactive_split_bytes_peak": 11855969792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 131522887680, + "reserved_bytes_peak": 131522887680, + "segment_count_current": 1946, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:15.038623Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": -597954560, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 866884096, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 13006536704, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 13006536704, + "segment_count_current": 1800, + "segment_count_current_delta": -146, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185317851553Z.pickle", + "timestamp_utc": "2026-08-26T18:53:17.905545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185317851553Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185317851553Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aec933926dac5a4082bf226d37340afb3b73f60a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185317851553Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:798616c9190edfa0d9516f09d82c0072df6ff11c8b0012fdad75c74ee509a5e2 +size 3223713 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185318232552Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185318232552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06403c89b5493567ca129b1dd73466d77c8c1a40 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185318232552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 3, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.232144Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 3, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:17.911571Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185318232552Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.243024Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185318232552Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185318232552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52353d88204695284a1f011d34f56338d273787d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185318232552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9897e3ba083cac6c6001887dbdd45e152462d0c616fcd11b02f900a7c957f18 +size 490754 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185318559283Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185318559283Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c900becc3a69e4b6bbd46892636593293b888569 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185318559283Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 85312985600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131077209088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 3, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.558901Z", + "world_size": 8 + }, + "allocated_bytes_current": 85312985600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123829451776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 85312985600, + "active_bytes_peak": 131077209088, + "allocated_bytes_current": 85312985600, + "allocated_bytes_peak": 123829451776, + "device": 3, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_peak": 12722853888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 144529424384, + "reserved_bytes_peak": 144529424384, + "segment_count_current": 1800, + "segment_count_peak": 1946, + "timestamp_utc": "2026-08-26T18:53:18.244175Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1492329984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12722853888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 144529424384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 144529424384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1800, + "segment_count_current_delta": 0, + "segment_count_peak": 1946, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185318559283Z.pickle", + "timestamp_utc": "2026-08-26T18:53:18.568732Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185318559283Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185318559283Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5e1943b0f61b0e168a9fac58dc2d9a5c58792c8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185318559283Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544db3d4c10b5c93d0645f1e7fb2eaf11c5e07e99e7ec9b7a66a9360d63b7ce1 +size 490754