diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c040d2b0748e436b41c51b425ea83f84b15ac1dc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,248 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 911 +1, 907 +2, 927 +3, 911 +4, 923 +5, 909 +6, 909 +7, 911 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 26289 +1, 26377 +2, 26223 +3, 26223 +4, 26223 +5, 26223 +6, 26355 +7, 26377 +0, 27983 +1, 27041 +2, 26889 +3, 26887 +4, 26887 +5, 27019 +6, 27409 +7, 27093 +0, 36841 +1, 36881 +2, 36701 +3, 36749 +4, 36773 +5, 36757 +6, 36919 +7, 36921 +0, 37785 +1, 37869 +2, 37719 +3, 37717 +4, 37717 +5, 37715 +6, 37851 +7, 37869 +0, 37873 +1, 37957 +2, 37807 +3, 37805 +4, 37805 +5, 37803 +6, 37939 +7, 37957 +0, 38079 +1, 38177 +2, 38011 +3, 37995 +4, 37991 +5, 37985 +6, 38129 +7, 38161 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38827 +1, 38911 +2, 38761 +3, 38761 +4, 38761 +5, 38757 +6, 38893 +7, 38911 +0, 38829 +1, 38913 +2, 38763 +3, 38763 +4, 38763 +5, 38759 +6, 38895 +7, 38913 +0, 112013 +1, 112081 +2, 111981 +3, 111985 +4, 111961 +5, 111957 +6, 112053 +7, 112085 +0, 112443 +1, 112529 +2, 112377 +3, 112377 +4, 112377 +5, 112375 +6, 112509 +7, 112529 +0, 112443 +1, 112529 +2, 112377 +3, 112377 +4, 112377 +5, 112375 +6, 112509 +7, 112529 +0, 112443 +1, 112529 +2, 112377 +3, 112377 +4, 112377 +5, 112375 +6, 112509 +7, 112529 +0, 112443 +1, 112529 +2, 112377 +3, 112377 +4, 112377 +5, 112375 +6, 112509 +7, 112529 +0, 112973 +1, 113085 +2, 112929 +3, 112941 +4, 112893 +5, 112881 +6, 113007 +7, 113043 +0, 115965 +1, 116051 +2, 115899 +3, 115899 +4, 115899 +5, 115897 +6, 116031 +7, 116051 +0, 117031 +1, 117079 +2, 116949 +3, 116937 +4, 116891 +5, 116953 +6, 117053 +7, 117061 +0, 63497 +1, 63309 +2, 68747 +3, 63297 +4, 63269 +5, 63207 +6, 64683 +7, 63311 +0, 151283 +1, 151215 +2, 151229 +3, 151189 +4, 151189 +5, 151061 +6, 151261 +7, 151215 +0, 150337 +1, 150333 +2, 150929 +3, 150563 +4, 150307 +5, 150761 +6, 150635 +7, 150269 +0, 777 +1, 1034 +2, 150281 +3, 2058 +4, 778 +5, 778 +6, 778 +7, 778 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87b7a5abc1492b8df2f79eeefd6e50bdba291189 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42487152128, + "active_bytes_current_delta": 19441586176, + "active_bytes_peak": 110640327680, + "active_bytes_peak_delta": 87582178816, + "after": { + "active_bytes_current": 42487152128, + "active_bytes_peak": 110640327680, + "allocated_bytes_current": 42487152128, + "allocated_bytes_peak": 110640327680, + "device": 0, + "inactive_split_bytes_current": 16369418752, + "inactive_split_bytes_peak": 28209760256, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115332874240, + "reserved_bytes_peak": 115332874240, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:01:23.031427Z", + "world_size": 8 + }, + "allocated_bytes_current": 42487152128, + "allocated_bytes_current_delta": 19441586176, + "allocated_bytes_peak": 110640327680, + "allocated_bytes_peak_delta": 87582178816, + "before": { + "active_bytes_current": 23045565952, + "active_bytes_peak": 23058148864, + "allocated_bytes_current": 23045565952, + "allocated_bytes_peak": 23058148864, + "device": 0, + "inactive_split_bytes_current": 352358912, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 25692209152, + "reserved_bytes_peak": 25692209152, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-26T19:00:42.978810Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16369418752, + "inactive_split_bytes_current_delta": 16017059840, + "inactive_split_bytes_peak": 28209760256, + "inactive_split_bytes_peak_delta": 27326524928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115332874240, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115332874240, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 922, + "segment_count_current_delta": 368, + "segment_count_peak": 922, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.114919Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1375901cb6c2aadaa780d800c8b1bf904f880781 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190123031908Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adac9585809518aeee8b590eb1dbac1b7b475832d71017a666d17a26187ae06c +size 4203011 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00fc7e96c257b673a9150188a63a5d44e6d8a86f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42487152128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124994119680, + "active_bytes_peak_delta": 14353792000, + "after": { + "active_bytes_current": 42487152128, + "active_bytes_peak": 124994119680, + "allocated_bytes_current": 42487152128, + "allocated_bytes_peak": 124855707648, + "device": 0, + "inactive_split_bytes_current": 16461693440, + "inactive_split_bytes_peak": 28547580416, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 127729139712, + "reserved_bytes_peak": 127729139712, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:01:23.760912Z", + "world_size": 8 + }, + "allocated_bytes_current": 42487152128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124855707648, + "allocated_bytes_peak_delta": 14215379968, + "before": { + "active_bytes_current": 42487152128, + "active_bytes_peak": 110640327680, + "allocated_bytes_current": 42487152128, + "allocated_bytes_peak": 110640327680, + "device": 0, + "inactive_split_bytes_current": 16369418752, + "inactive_split_bytes_peak": 28209760256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115332874240, + "reserved_bytes_peak": 115332874240, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:01:23.127117Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16461693440, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28547580416, + "inactive_split_bytes_peak_delta": 337820160, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 127729139712, + "reserved_bytes_current_delta": 12396265472, + "reserved_bytes_peak": 127729139712, + "reserved_bytes_peak_delta": 12396265472, + "segment_count_current": 933, + "segment_count_current_delta": 11, + "segment_count_peak": 933, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.838750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48164d376fec178e46eba7fbfd29bc70b1d302c6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190123761329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:695bf7566f0ca55c666db0475079c916bca930ca2e25b6aba6f827de5871eb47 +size 4587773 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..772e3f88e5d5d167c4e6daaa39c0e3c23b70e02e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42487152128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125133957120, + "active_bytes_peak_delta": 139837440, + "after": { + "active_bytes_current": 42487152128, + "active_bytes_peak": 125133957120, + "allocated_bytes_current": 42487152128, + "allocated_bytes_peak": 124857133056, + "device": 0, + "inactive_split_bytes_current": 16461693440, + "inactive_split_bytes_peak": 29876894208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 127729139712, + "reserved_bytes_peak": 127729139712, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:01:24.484602Z", + "world_size": 8 + }, + "allocated_bytes_current": 42487152128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124857133056, + "allocated_bytes_peak_delta": 1425408, + "before": { + "active_bytes_current": 42487152128, + "active_bytes_peak": 124994119680, + "allocated_bytes_current": 42487152128, + "allocated_bytes_peak": 124855707648, + "device": 0, + "inactive_split_bytes_current": 16461693440, + "inactive_split_bytes_peak": 28547580416, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 127729139712, + "reserved_bytes_peak": 127729139712, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:01:23.848707Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16461693440, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 29876894208, + "inactive_split_bytes_peak_delta": 1329313792, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 127729139712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127729139712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.559979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa6e50b5ba42125f55f514aa8d5e8d6f637cb273 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190124485008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e91880282c9511ee39ff513da9b5592529c634c7bc780ecb33462c24b71b5f1d +size 4605081 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8589ea4dbef8e221478e4d330fc6388549d6f78a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505113088, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127972926464, + "active_bytes_peak_delta": 2838969344, + "after": { + "active_bytes_current": 42505113088, + "active_bytes_peak": 127972926464, + "allocated_bytes_current": 42505113088, + "allocated_bytes_peak": 124857133056, + "device": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_peak": 43042810368, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_peak": 151074635776, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:01:26.509405Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505113088, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124857133056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42487152128, + "active_bytes_peak": 125133957120, + "allocated_bytes_current": 42487152128, + "allocated_bytes_peak": 124857133056, + "device": 0, + "inactive_split_bytes_current": 16461693440, + "inactive_split_bytes_peak": 29876894208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 127729139712, + "reserved_bytes_peak": 127729139712, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:01:24.569517Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 43042810368, + "inactive_split_bytes_peak_delta": 13165916160, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_current_delta": 23345496064, + "reserved_bytes_peak": 151074635776, + "reserved_bytes_peak_delta": 23345496064, + "segment_count_current": 1130, + "segment_count_current_delta": 197, + "segment_count_peak": 1130, + "segment_count_peak_delta": 197, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.596160Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf566833c56bcf0fc3ca7eb2fbc0b9f0ca1d78ee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190126510019Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07ffd7cfe481a42d34cd7fbc78f6abb67a51083535c6b5ce422c0b0e07b4576e +size 4739794 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b2a2017068c1aeb4d7b3cb8bde26e46f48120c5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505113088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127972926464, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42505113088, + "active_bytes_peak": 127972926464, + "allocated_bytes_current": 42505113088, + "allocated_bytes_peak": 124857133056, + "device": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_peak": 43042810368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_peak": 151074635776, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:01:27.191302Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505113088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124857133056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42505113088, + "active_bytes_peak": 127972926464, + "allocated_bytes_current": 42505113088, + "allocated_bytes_peak": 124857133056, + "device": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_peak": 43042810368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_peak": 151074635776, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:01:26.606920Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 43042810368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151074635776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1130, + "segment_count_current_delta": 0, + "segment_count_peak": 1130, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d4ceecb6c3eef319653fd9a0cb54516909c7fc9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190127191709Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bfa5cbc889b82e95faa85d9e523b727238f1706c44b32b7ac1b134ddd33f250 +size 383472 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.json new file mode 100644 index 0000000000000000000000000000000000000000..671843621906ab5cb36a31ad0813dfba4cfb1f8e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505113088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127972926464, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42505113088, + "active_bytes_peak": 127972926464, + "allocated_bytes_current": 42505113088, + "allocated_bytes_peak": 124857133056, + "device": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_peak": 43042810368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_peak": 151074635776, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:01:27.792453Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505113088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124857133056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42505113088, + "active_bytes_peak": 127972926464, + "allocated_bytes_current": 42505113088, + "allocated_bytes_peak": 124857133056, + "device": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_peak": 43042810368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_peak": 151074635776, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:01:27.203084Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18522010112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 43042810368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 151074635776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151074635776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1130, + "segment_count_current_delta": 0, + "segment_count_peak": 1130, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.802293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85f24cd3bb8b8b625ed53cc062cbafa8da7a0966 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190127792841Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0fce639d03a24fcadde45bffc3e5c620abacd21544d6729772208d908860dc2 +size 383472 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1fec4341f61f0ee2b8e1ea870176a28486a0842b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42495456768, + "active_bytes_current_delta": 19447151104, + "active_bytes_peak": 110662637568, + "active_bytes_peak_delta": 87601748992, + "after": { + "active_bytes_current": 42495456768, + "active_bytes_peak": 110662637568, + "allocated_bytes_current": 42495456768, + "allocated_bytes_peak": 110662637568, + "device": 1, + "inactive_split_bytes_current": 16382085632, + "inactive_split_bytes_peak": 28267021312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115423051776, + "reserved_bytes_peak": 115423051776, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:01:23.031572Z", + "world_size": 8 + }, + "allocated_bytes_current": 42495456768, + "allocated_bytes_current_delta": 19447151104, + "allocated_bytes_peak": 110662637568, + "allocated_bytes_peak_delta": 87601748992, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 1, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:00:42.978679Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16382085632, + "inactive_split_bytes_current_delta": 15879374336, + "inactive_split_bytes_peak": 28267021312, + "inactive_split_bytes_peak_delta": 27383785984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115423051776, + "reserved_bytes_current_delta": 89638567936, + "reserved_bytes_peak": 115423051776, + "reserved_bytes_peak_delta": 89638567936, + "segment_count_current": 925, + "segment_count_current_delta": 367, + "segment_count_peak": 925, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.113479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7cb429514c0568cc9b92baccd20b22219bf3c264 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190123032066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae1e8a691184138cf45ed9c4cfd257c5a59a7cfc0efa87846d81383e41ac8205 +size 4145363 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d64743ddddfc9e8ca73affc92ec88324947235f9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42495456768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124990884864, + "active_bytes_peak_delta": 14328247296, + "after": { + "active_bytes_current": 42495456768, + "active_bytes_peak": 124990884864, + "allocated_bytes_current": 42495456768, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 16474360320, + "inactive_split_bytes_peak": 28629453312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:23.760843Z", + "world_size": 8 + }, + "allocated_bytes_current": 42495456768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124852472832, + "allocated_bytes_peak_delta": 14189835264, + "before": { + "active_bytes_current": 42495456768, + "active_bytes_peak": 110662637568, + "allocated_bytes_current": 42495456768, + "allocated_bytes_peak": 110662637568, + "device": 1, + "inactive_split_bytes_current": 16382085632, + "inactive_split_bytes_peak": 28267021312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115423051776, + "reserved_bytes_peak": 115423051776, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:01:23.124351Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16474360320, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28629453312, + "inactive_split_bytes_peak_delta": 362432000, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127815122944, + "reserved_bytes_current_delta": 12392071168, + "reserved_bytes_peak": 127815122944, + "reserved_bytes_peak_delta": 12392071168, + "segment_count_current": 934, + "segment_count_current_delta": 9, + "segment_count_peak": 934, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.834929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64525e87603b3238bdce0bea72a9f1241351b4e8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190123761255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a75fa12914093db306eb848547efb9a8b75549cbf2f17fbe222156468bf99657 +size 4529532 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.json new file mode 100644 index 0000000000000000000000000000000000000000..762db175a23d2040afabea5cc7b741a04c9a52da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42495456768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125128502272, + "active_bytes_peak_delta": 137617408, + "after": { + "active_bytes_current": 42495456768, + "active_bytes_peak": 125128502272, + "allocated_bytes_current": 42495456768, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 16382085632, + "inactive_split_bytes_peak": 28629453312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:24.484674Z", + "world_size": 8 + }, + "allocated_bytes_current": 42495456768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124852472832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42495456768, + "active_bytes_peak": 124990884864, + "allocated_bytes_current": 42495456768, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 16474360320, + "inactive_split_bytes_peak": 28629453312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:23.844050Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16382085632, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28629453312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127815122944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127815122944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.558894Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85aa9716e97ef3002aa15d1c29e7908d623e86b2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190124485076Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7c57e637480c1be5c13129517958afc9cf9edf24f6a11e1f8a0ba7ff2193256 +size 4546981 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4e6a84562b61ad85d0d91571e6b32bb5ab90fbb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42513417728, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127981231104, + "active_bytes_peak_delta": 2852728832, + "after": { + "active_bytes_current": 42513417728, + "active_bytes_peak": 127981231104, + "allocated_bytes_current": 42513417728, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_peak": 42956911104, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:26.509381Z", + "world_size": 8 + }, + "allocated_bytes_current": 42513417728, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124852472832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42495456768, + "active_bytes_peak": 125128502272, + "allocated_bytes_current": 42495456768, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 16382085632, + "inactive_split_bytes_peak": 28629453312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:24.567614Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 42956911104, + "inactive_split_bytes_peak_delta": 14327457792, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 151003332608, + "reserved_bytes_current_delta": 23188209664, + "reserved_bytes_peak": 151003332608, + "reserved_bytes_peak_delta": 23188209664, + "segment_count_current": 1099, + "segment_count_current_delta": 165, + "segment_count_peak": 1099, + "segment_count_peak_delta": 165, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.594279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..279f05a2ac6c2eeead209c9f4aebb04e7db495de --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190126509869Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59eaaf79332db41fce0df031c651253d496b57ff4f14951a4684376391ea3934 +size 4764032 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36267712f0f9af02e5ad3df5612ffbb765728709 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42513417728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127981231104, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42513417728, + "active_bytes_peak": 127981231104, + "allocated_bytes_current": 42513417728, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_peak": 42956911104, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:27.191308Z", + "world_size": 8 + }, + "allocated_bytes_current": 42513417728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124852472832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42513417728, + "active_bytes_peak": 127981231104, + "allocated_bytes_current": 42513417728, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_peak": 42956911104, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:26.604257Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42956911104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 151003332608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151003332608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1099, + "segment_count_current_delta": 0, + "segment_count_peak": 1099, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201474Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8c9817fe01e489b4bdbf8c7f8a1b87b6f668ea27 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190127191708Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c1e11ca04bb16997c2d624984b6ace9dc226f4ae5482e86409ddc39b3e32e38 +size 405776 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fef4f756a8a63dc661365ca23886b90764e9c352 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42513417728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127981231104, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42513417728, + "active_bytes_peak": 127981231104, + "allocated_bytes_current": 42513417728, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_peak": 42956911104, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:27.792496Z", + "world_size": 8 + }, + "allocated_bytes_current": 42513417728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124852472832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42513417728, + "active_bytes_peak": 127981231104, + "allocated_bytes_current": 42513417728, + "allocated_bytes_peak": 124852472832, + "device": 1, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_peak": 42956911104, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:27.202606Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18534676992, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42956911104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 151003332608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151003332608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1099, + "segment_count_current_delta": 0, + "segment_count_peak": 1099, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.802691Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51cbd3332e70473e12b8fa519917091f4cbe9a09 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190127792897Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f190610ca87db8b2535504aa7ad66a6c3aa4957dd522af68693ad66b4202f32 +size 405776 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71f0b5cb94644f5da4ce66219f46e9903cea80aa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42478725120, + "active_bytes_current_delta": 19427855360, + "active_bytes_peak": 110641677312, + "active_bytes_peak_delta": 87578224640, + "after": { + "active_bytes_current": 42478725120, + "active_bytes_peak": 110641677312, + "allocated_bytes_current": 42478725120, + "allocated_bytes_peak": 110641677312, + "device": 2, + "inactive_split_bytes_current": 16319125504, + "inactive_split_bytes_peak": 28242721792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:01:23.031410Z", + "world_size": 8 + }, + "allocated_bytes_current": 42478725120, + "allocated_bytes_current_delta": 19427855360, + "allocated_bytes_peak": 110641677312, + "allocated_bytes_peak_delta": 87578224640, + "before": { + "active_bytes_current": 23050869760, + "active_bytes_peak": 23063452672, + "allocated_bytes_current": 23050869760, + "allocated_bytes_peak": 23063452672, + "device": 2, + "inactive_split_bytes_current": 363832320, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:00:42.978802Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16319125504, + "inactive_split_bytes_current_delta": 15955293184, + "inactive_split_bytes_peak": 28242721792, + "inactive_split_bytes_peak_delta": 27359486464, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115263668224, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115263668224, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 919, + "segment_count_current_delta": 368, + "segment_count_peak": 919, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.115545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab58c775c30f587e0fc07cefd50fabfc15f951f8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190123031905Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e86fb2ca11ce46c42d117498573ab954f8e0ba6596e1ede0faef8c4c1998b51 +size 4127102 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9665822cb673fd8440ca705bf4e268ad7abf007 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42478725120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124971887616, + "active_bytes_peak_delta": 14330210304, + "after": { + "active_bytes_current": 42478725120, + "active_bytes_peak": 124971887616, + "allocated_bytes_current": 42478725120, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 16411400192, + "inactive_split_bytes_peak": 28242721792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127662030848, + "reserved_bytes_peak": 127662030848, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:01:23.760848Z", + "world_size": 8 + }, + "allocated_bytes_current": 42478725120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124833475584, + "allocated_bytes_peak_delta": 14191798272, + "before": { + "active_bytes_current": 42478725120, + "active_bytes_peak": 110641677312, + "allocated_bytes_current": 42478725120, + "allocated_bytes_peak": 110641677312, + "device": 2, + "inactive_split_bytes_current": 16319125504, + "inactive_split_bytes_peak": 28242721792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:01:23.127162Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16411400192, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28242721792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127662030848, + "reserved_bytes_current_delta": 12398362624, + "reserved_bytes_peak": 127662030848, + "reserved_bytes_peak_delta": 12398362624, + "segment_count_current": 931, + "segment_count_current_delta": 12, + "segment_count_peak": 931, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.837243Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de445faf72150cbb61c22b969c9c37c6fe04da83 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190123761256Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4e3a7ba70cad4ed95258c7610b044f2cdf5c9676e70ba3c3ff81a7ffcd3f8a3 +size 4509234 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2be5c9321e4dfd0e74a5a5c27a9e7e862eaa2e32 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42478725120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125107940864, + "active_bytes_peak_delta": 136053248, + "after": { + "active_bytes_current": 42478725120, + "active_bytes_peak": 125107940864, + "allocated_bytes_current": 42478725120, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 16413497344, + "inactive_split_bytes_peak": 28242721792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127754305536, + "reserved_bytes_peak": 127754305536, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:01:24.484587Z", + "world_size": 8 + }, + "allocated_bytes_current": 42478725120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124833475584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42478725120, + "active_bytes_peak": 124971887616, + "allocated_bytes_current": 42478725120, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 16411400192, + "inactive_split_bytes_peak": 28242721792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127662030848, + "reserved_bytes_peak": 127662030848, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:01:23.846392Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16413497344, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 28242721792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127754305536, + "reserved_bytes_current_delta": 92274688, + "reserved_bytes_peak": 127754305536, + "reserved_bytes_peak_delta": 92274688, + "segment_count_current": 932, + "segment_count_current_delta": 1, + "segment_count_peak": 932, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.559130Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94c3ea8fe13e73aff0c60296c15a643d53e0dd97 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190124484977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:383a5aee6269d371c8d80d2e95f9992e08158c80891087fa1ed8936cf99b6170 +size 4526520 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa48194176f03a668effc00d59b8f370a8f42cd0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496686080, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127964499456, + "active_bytes_peak_delta": 2856558592, + "after": { + "active_bytes_current": 42496686080, + "active_bytes_peak": 127964499456, + "allocated_bytes_current": 42496686080, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_peak": 42998808576, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 151018012672, + "reserved_bytes_peak": 151018012672, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:01:26.509398Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496686080, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124833475584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42478725120, + "active_bytes_peak": 125107940864, + "allocated_bytes_current": 42478725120, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 16413497344, + "inactive_split_bytes_peak": 28242721792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127754305536, + "reserved_bytes_peak": 127754305536, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:01:24.567968Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_current_delta": 2062413824, + "inactive_split_bytes_peak": 42998808576, + "inactive_split_bytes_peak_delta": 14756086784, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 151018012672, + "reserved_bytes_current_delta": 23263707136, + "reserved_bytes_peak": 151018012672, + "reserved_bytes_peak_delta": 23263707136, + "segment_count_current": 1133, + "segment_count_current_delta": 201, + "segment_count_peak": 1133, + "segment_count_peak_delta": 201, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.595370Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09639b747c4fbeaf758511ca679ca7cf63236cce --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190126509999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8abaa17e2a9bd1c4298e1c241b78780bebb5b507fc866cf26c32c3cc4196c8a5 +size 4737545 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6048e45556cc60bcde70fdc17bd791447745455 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496686080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127964499456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42496686080, + "active_bytes_peak": 127964499456, + "allocated_bytes_current": 42496686080, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_peak": 42998808576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 151018012672, + "reserved_bytes_peak": 151018012672, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:01:27.191302Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496686080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124833475584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42496686080, + "active_bytes_peak": 127964499456, + "allocated_bytes_current": 42496686080, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_peak": 42998808576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 151018012672, + "reserved_bytes_peak": 151018012672, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:01:26.605464Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42998808576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 151018012672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151018012672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201722Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51079f8f098ec7ae2f99a6b6bb3171b67697ec5f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190127191697Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1450544ece0e239156314d91696b8be88c18a449024e0ee7ed120fe2ae44a393 +size 382821 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae64c7d0a6efd1d3ccf8d8a84214028d42673fc4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496686080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127964499456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42496686080, + "active_bytes_peak": 127964499456, + "allocated_bytes_current": 42496686080, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_peak": 42998808576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 151018012672, + "reserved_bytes_peak": 151018012672, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:01:27.792458Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496686080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124833475584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42496686080, + "active_bytes_peak": 127964499456, + "allocated_bytes_current": 42496686080, + "allocated_bytes_peak": 124833475584, + "device": 2, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_peak": 42998808576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 151018012672, + "reserved_bytes_peak": 151018012672, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:01:27.202836Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18475911168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42998808576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 151018012672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151018012672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.802004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e32ccc873138bea9ec5a578aa5a76d1a27114aaf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190127792839Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:166f9a52a0c266d251cb976688afc69a55d1e67de9721b7d5b8b76c4284f6a93 +size 382821 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.json new file mode 100644 index 0000000000000000000000000000000000000000..340e8a4ad975dd18f96cd47dfe2962d2defca13a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42492211712, + "active_bytes_current_delta": 19443268608, + "active_bytes_peak": 110661306368, + "active_bytes_peak_delta": 87599780352, + "after": { + "active_bytes_current": 42492211712, + "active_bytes_peak": 110661306368, + "allocated_bytes_current": 42492211712, + "allocated_bytes_peak": 110661306368, + "device": 3, + "inactive_split_bytes_current": 16263695872, + "inactive_split_bytes_peak": 28274912256, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:01:23.031470Z", + "world_size": 8 + }, + "allocated_bytes_current": 42492211712, + "allocated_bytes_current_delta": 19443268608, + "allocated_bytes_peak": 110661306368, + "allocated_bytes_peak_delta": 87599780352, + "before": { + "active_bytes_current": 23048943104, + "active_bytes_peak": 23061526016, + "allocated_bytes_current": 23048943104, + "allocated_bytes_peak": 23061526016, + "device": 3, + "inactive_split_bytes_current": 468519424, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:00:42.978699Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16263695872, + "inactive_split_bytes_current_delta": 15795176448, + "inactive_split_bytes_peak": 28274912256, + "inactive_split_bytes_peak_delta": 27391676928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115263668224, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115263668224, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 919, + "segment_count_current_delta": 368, + "segment_count_peak": 919, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.114264Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7379fb9c4d77781dcdee7b24f8f25294e9ef53a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190123031937Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caeb44b6301ce1a7c98ad82eb2306e870843f0c4a8305c956f996fa8bbf84aa0 +size 4167516 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.json new file mode 100644 index 0000000000000000000000000000000000000000..413b03a7e88e5ed49f660255a7046912fb005c49 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42492211712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124978370560, + "active_bytes_peak_delta": 14317064192, + "after": { + "active_bytes_current": 42492211712, + "active_bytes_peak": 124978370560, + "allocated_bytes_current": 42492211712, + "allocated_bytes_peak": 124839958528, + "device": 3, + "inactive_split_bytes_current": 16358067712, + "inactive_split_bytes_peak": 28274912256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 129805320192, + "reserved_bytes_peak": 129805320192, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-26T19:01:23.760842Z", + "world_size": 8 + }, + "allocated_bytes_current": 42492211712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124839958528, + "allocated_bytes_peak_delta": 14178652160, + "before": { + "active_bytes_current": 42492211712, + "active_bytes_peak": 110661306368, + "allocated_bytes_current": 42492211712, + "allocated_bytes_peak": 110661306368, + "device": 3, + "inactive_split_bytes_current": 16263695872, + "inactive_split_bytes_peak": 28274912256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:01:23.125881Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16358067712, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 28274912256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 129805320192, + "reserved_bytes_current_delta": 14541651968, + "reserved_bytes_peak": 129805320192, + "reserved_bytes_peak_delta": 14541651968, + "segment_count_current": 930, + "segment_count_current_delta": 11, + "segment_count_peak": 930, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.836274Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90630c3c6714bc851bddab82fd0ef007241de4b2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190123761256Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:921102b11c2112f3c18c026a0dd9768cf606509d3076f194158dfc02249ac88a +size 4585585 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6b403a2d9bb2b35204f5f108348e8f3ea6379fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42492211712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125118871552, + "active_bytes_peak_delta": 140500992, + "after": { + "active_bytes_current": 42492211712, + "active_bytes_peak": 125118871552, + "allocated_bytes_current": 42492211712, + "allocated_bytes_peak": 124842047488, + "device": 3, + "inactive_split_bytes_current": 16358067712, + "inactive_split_bytes_peak": 28274912256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 129805320192, + "reserved_bytes_peak": 129805320192, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-26T19:01:24.484592Z", + "world_size": 8 + }, + "allocated_bytes_current": 42492211712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124842047488, + "allocated_bytes_peak_delta": 2088960, + "before": { + "active_bytes_current": 42492211712, + "active_bytes_peak": 124978370560, + "allocated_bytes_current": 42492211712, + "allocated_bytes_peak": 124839958528, + "device": 3, + "inactive_split_bytes_current": 16358067712, + "inactive_split_bytes_peak": 28274912256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 129805320192, + "reserved_bytes_peak": 129805320192, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-26T19:01:23.845623Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16358067712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28274912256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 129805320192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129805320192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 930, + "segment_count_current_delta": 0, + "segment_count_peak": 930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.560189Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bf886d6f4cabae924663a0b449f216a176c8679 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190124485004Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f433d080b92a226a5558d1c2b013896ff57a42e10767bbc0b0e0b106cf54458b +size 4572603 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2b38083b7eb61c7c60dd33b22108109e269dcfa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42510172672, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127977986048, + "active_bytes_peak_delta": 2859114496, + "after": { + "active_bytes_current": 42510172672, + "active_bytes_peak": 127977986048, + "allocated_bytes_current": 42510172672, + "allocated_bytes_peak": 124842047488, + "device": 3, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_peak": 42846909952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:26.509376Z", + "world_size": 8 + }, + "allocated_bytes_current": 42510172672, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124842047488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42492211712, + "active_bytes_peak": 125118871552, + "allocated_bytes_current": 42492211712, + "allocated_bytes_peak": 124842047488, + "device": 3, + "inactive_split_bytes_current": 16358067712, + "inactive_split_bytes_peak": 28274912256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 129805320192, + "reserved_bytes_peak": 129805320192, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-26T19:01:24.568936Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 42846909952, + "inactive_split_bytes_peak_delta": 14571997696, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150976069632, + "reserved_bytes_current_delta": 21170749440, + "reserved_bytes_peak": 150976069632, + "reserved_bytes_peak_delta": 21170749440, + "segment_count_current": 1113, + "segment_count_current_delta": 183, + "segment_count_peak": 1113, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.594682Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a08c71cb884db315b13fbdc6b99e5592c0008b9a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190126509865Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925adec3bbdc894d9b03b4822e74824f84c6869ad3a9e54dc7e09228e1a1fda8 +size 4733954 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.json new file mode 100644 index 0000000000000000000000000000000000000000..996e2d0538815379e953a29559d65efdc62389ee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42510172672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127977986048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42510172672, + "active_bytes_peak": 127977986048, + "allocated_bytes_current": 42510172672, + "allocated_bytes_peak": 124842047488, + "device": 3, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_peak": 42846909952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:27.191276Z", + "world_size": 8 + }, + "allocated_bytes_current": 42510172672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124842047488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42510172672, + "active_bytes_peak": 127977986048, + "allocated_bytes_current": 42510172672, + "allocated_bytes_peak": 124842047488, + "device": 3, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_peak": 42846909952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:26.606379Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42846909952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150976069632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150976069632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1113, + "segment_count_current_delta": 0, + "segment_count_peak": 1113, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44a99c61c55bf85f01de74e0a75701dd40cad53f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190127191681Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a66ee1dcece15c4ed708c15314ed0b969cadee97d4e9036ab8a85455cd2a6a45 +size 376846 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dda706ee6f974ef90e25279a1c5ec734c4049a62 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42510172672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127977986048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42510172672, + "active_bytes_peak": 127977986048, + "allocated_bytes_current": 42510172672, + "allocated_bytes_peak": 124842047488, + "device": 3, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_peak": 42846909952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:27.792498Z", + "world_size": 8 + }, + "allocated_bytes_current": 42510172672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124842047488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42510172672, + "active_bytes_peak": 127977986048, + "allocated_bytes_current": 42510172672, + "allocated_bytes_peak": 124842047488, + "device": 3, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_peak": 42846909952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:27.202580Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18418384384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42846909952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150976069632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150976069632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1113, + "segment_count_current_delta": 0, + "segment_count_peak": 1113, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.802750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d6026620f6ab4848d072cd6f291a70a8ee7900e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190127792900Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d71623ea70e2bd1904739534d921c7c091502d45bd9de68e24869bddb6274cc7 +size 376898 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ea9c92701ff27789bdb41bbc796b5ceac8baeed --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496943104, + "active_bytes_current_delta": 19448559616, + "active_bytes_peak": 110649736192, + "active_bytes_peak_delta": 87588769792, + "after": { + "active_bytes_current": 42496943104, + "active_bytes_peak": 110649736192, + "allocated_bytes_current": 42496943104, + "allocated_bytes_peak": 110649736192, + "device": 4, + "inactive_split_bytes_current": 16261061632, + "inactive_split_bytes_peak": 28203455488, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:01:23.031504Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496943104, + "allocated_bytes_current_delta": 19448559616, + "allocated_bytes_peak": 110649736192, + "allocated_bytes_peak_delta": 87588769792, + "before": { + "active_bytes_current": 23048383488, + "active_bytes_peak": 23060966400, + "allocated_bytes_current": 23048383488, + "allocated_bytes_peak": 23060966400, + "device": 4, + "inactive_split_bytes_current": 376804352, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:00:42.978679Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16261061632, + "inactive_split_bytes_current_delta": 15884257280, + "inactive_split_bytes_peak": 28203455488, + "inactive_split_bytes_peak_delta": 27320220160, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115263668224, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115263668224, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 919, + "segment_count_current_delta": 368, + "segment_count_peak": 919, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.114526Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d6277ac2361cce04316b3d3d2007cf3b655be00 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190123031984Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e895a0229ef7e89ff68913ad4c3c311f9c023f55868c7469fc38632fa5f4b53a +size 4121434 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190123761385Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190123761385Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7868714317897ac071a22f41064abe5f670385ea --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190123761385Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496943104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124998892544, + "active_bytes_peak_delta": 14349156352, + "after": { + "active_bytes_current": 42496943104, + "active_bytes_peak": 124998892544, + "allocated_bytes_current": 42496943104, + "allocated_bytes_peak": 124860480512, + "device": 4, + "inactive_split_bytes_current": 16353336320, + "inactive_split_bytes_peak": 28203455488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127657836544, + "reserved_bytes_peak": 127657836544, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-26T19:01:23.760956Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496943104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124860480512, + "allocated_bytes_peak_delta": 14210744320, + "before": { + "active_bytes_current": 42496943104, + "active_bytes_peak": 110649736192, + "allocated_bytes_current": 42496943104, + "allocated_bytes_peak": 110649736192, + "device": 4, + "inactive_split_bytes_current": 16261061632, + "inactive_split_bytes_peak": 28203455488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:01:23.125156Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16353336320, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28203455488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127657836544, + "reserved_bytes_current_delta": 12394168320, + "reserved_bytes_peak": 127657836544, + "reserved_bytes_peak_delta": 12394168320, + "segment_count_current": 929, + "segment_count_current_delta": 10, + "segment_count_peak": 929, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190123761385Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.833571Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190123761385Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190123761385Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4615ed2dc7c26f885da2fadbe98ea7ba12db7df6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190123761385Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900e45629d4385a551403f9459261ac354457c4f162ba035b16f774f6f0c158f +size 4505666 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190124485043Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190124485043Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc8b42245891ad47c1124f9948e2649085739a0c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190124485043Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496943104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125139909632, + "active_bytes_peak_delta": 141017088, + "after": { + "active_bytes_current": 42496943104, + "active_bytes_peak": 125139909632, + "allocated_bytes_current": 42496943104, + "allocated_bytes_peak": 124863085568, + "device": 4, + "inactive_split_bytes_current": 16353336320, + "inactive_split_bytes_peak": 28203455488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127657836544, + "reserved_bytes_peak": 127657836544, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-26T19:01:24.484637Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496943104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124863085568, + "allocated_bytes_peak_delta": 2605056, + "before": { + "active_bytes_current": 42496943104, + "active_bytes_peak": 124998892544, + "allocated_bytes_current": 42496943104, + "allocated_bytes_peak": 124860480512, + "device": 4, + "inactive_split_bytes_current": 16353336320, + "inactive_split_bytes_peak": 28203455488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127657836544, + "reserved_bytes_peak": 127657836544, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-26T19:01:23.841590Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16353336320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28203455488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127657836544, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127657836544, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 929, + "segment_count_current_delta": 0, + "segment_count_peak": 929, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190124485043Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.557627Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190124485043Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190124485043Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e9453d79290786dc1a3f055f5ea5cbe12373618 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190124485043Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea99176c4dcaebda90f3a60119a9c1048c708c3121296865a2cab361b07d6634 +size 4521768 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190126509828Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190126509828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a47e78790c504e442a460d7299e92eaed46d5af --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190126509828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42514904064, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127982717440, + "active_bytes_peak_delta": 2842807808, + "after": { + "active_bytes_current": 42514904064, + "active_bytes_peak": 127982717440, + "allocated_bytes_current": 42514904064, + "allocated_bytes_peak": 124863085568, + "device": 4, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_peak": 42752001024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:26.509376Z", + "world_size": 8 + }, + "allocated_bytes_current": 42514904064, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124863085568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42496943104, + "active_bytes_peak": 125139909632, + "allocated_bytes_current": 42496943104, + "allocated_bytes_peak": 124863085568, + "device": 4, + "inactive_split_bytes_current": 16353336320, + "inactive_split_bytes_peak": 28203455488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127657836544, + "reserved_bytes_peak": 127657836544, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-26T19:01:24.565619Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 42752001024, + "inactive_split_bytes_peak_delta": 14548545536, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150976069632, + "reserved_bytes_current_delta": 23318233088, + "reserved_bytes_peak": 150976069632, + "reserved_bytes_peak_delta": 23318233088, + "segment_count_current": 1113, + "segment_count_current_delta": 184, + "segment_count_peak": 1113, + "segment_count_peak_delta": 184, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190126509828Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.592550Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190126509828Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190126509828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e11c74f07f16579c6f6976d7e600b99bef415408 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190126509828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09dbe247a091cada5a30e824cf00256bafa81742b212ef6650b6a5bb6a6c24b7 +size 4736383 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190127191722Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190127191722Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b54baf581e27944352f22cfee610d3ade3b9ae09 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190127191722Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42514904064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127982717440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42514904064, + "active_bytes_peak": 127982717440, + "allocated_bytes_current": 42514904064, + "allocated_bytes_peak": 124863085568, + "device": 4, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_peak": 42752001024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:27.191312Z", + "world_size": 8 + }, + "allocated_bytes_current": 42514904064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124863085568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42514904064, + "active_bytes_peak": 127982717440, + "allocated_bytes_current": 42514904064, + "allocated_bytes_peak": 124863085568, + "device": 4, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_peak": 42752001024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:26.601944Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42752001024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150976069632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150976069632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1113, + "segment_count_current_delta": 0, + "segment_count_peak": 1113, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190127191722Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201329Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190127191722Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190127191722Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a679fe40bde5793b616db3854d1239afd5805cdd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190127191722Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4873e6a02c793a7077e5370cdd801d60da7b7040068a547c0e5eeb2cb568a08d +size 379898 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190127792948Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190127792948Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c29d545dff63b436a830d392912fc77ba4b2629a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190127792948Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42514904064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127982717440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42514904064, + "active_bytes_peak": 127982717440, + "allocated_bytes_current": 42514904064, + "allocated_bytes_peak": 124863085568, + "device": 4, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_peak": 42752001024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:27.792521Z", + "world_size": 8 + }, + "allocated_bytes_current": 42514904064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124863085568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42514904064, + "active_bytes_peak": 127982717440, + "allocated_bytes_current": 42514904064, + "allocated_bytes_peak": 124863085568, + "device": 4, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_peak": 42752001024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150976069632, + "reserved_bytes_peak": 150976069632, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:01:27.202438Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18413652992, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42752001024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150976069632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150976069632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1113, + "segment_count_current_delta": 0, + "segment_count_peak": 1113, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190127792948Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.803887Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190127792948Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190127792948Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e558d912652eef8fd80ca71391ddf2f813cfb42f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190127792948Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f7a77b5e6a3419da020a5fbaa5162d1a4d8280274ff0fdf0293fb8776a8f046 +size 379898 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190123031907Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190123031907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e42a17088a5989b3c97919376dd5278b8453bc8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190123031907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42489992192, + "active_bytes_current_delta": 19436809216, + "active_bytes_peak": 110651151360, + "active_bytes_peak_delta": 87585385472, + "after": { + "active_bytes_current": 42489992192, + "active_bytes_peak": 110651151360, + "allocated_bytes_current": 42489992192, + "allocated_bytes_peak": 110651151360, + "device": 5, + "inactive_split_bytes_current": 16226069504, + "inactive_split_bytes_peak": 28322138112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115261571072, + "reserved_bytes_peak": 115261571072, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:01:23.031396Z", + "world_size": 8 + }, + "allocated_bytes_current": 42489992192, + "allocated_bytes_current_delta": 19436809216, + "allocated_bytes_peak": 110651151360, + "allocated_bytes_peak_delta": 87585385472, + "before": { + "active_bytes_current": 23053182976, + "active_bytes_peak": 23065765888, + "allocated_bytes_current": 23053182976, + "allocated_bytes_peak": 23065765888, + "device": 5, + "inactive_split_bytes_current": 462182400, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:00:42.978740Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16226069504, + "inactive_split_bytes_current_delta": 15763887104, + "inactive_split_bytes_peak": 28322138112, + "inactive_split_bytes_peak_delta": 27438902784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115261571072, + "reserved_bytes_current_delta": 89638567936, + "reserved_bytes_peak": 115261571072, + "reserved_bytes_peak_delta": 89638567936, + "segment_count_current": 918, + "segment_count_current_delta": 367, + "segment_count_peak": 918, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190123031907Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.113775Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190123031907Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190123031907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b5ab3d1adbdbfc4dbc17f7bf57f1e419a514413 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190123031907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5068ec8c332d809b3f0aa5c61211de4e10a79b100eacb9b17bc0da1a3c7f029 +size 4144185 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190123761250Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190123761250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa9fffa7ae5be4278be54a738fc2e6c7ee96a6a6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190123761250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42489992192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124978663424, + "active_bytes_peak_delta": 14327512064, + "after": { + "active_bytes_current": 42489992192, + "active_bytes_peak": 124978663424, + "allocated_bytes_current": 42489992192, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 16318344192, + "inactive_split_bytes_peak": 28322138112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127653642240, + "reserved_bytes_peak": 127653642240, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:01:23.760843Z", + "world_size": 8 + }, + "allocated_bytes_current": 42489992192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124840251392, + "allocated_bytes_peak_delta": 14189100032, + "before": { + "active_bytes_current": 42489992192, + "active_bytes_peak": 110651151360, + "allocated_bytes_current": 42489992192, + "allocated_bytes_peak": 110651151360, + "device": 5, + "inactive_split_bytes_current": 16226069504, + "inactive_split_bytes_peak": 28322138112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115261571072, + "reserved_bytes_peak": 115261571072, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:01:23.125693Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16318344192, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28322138112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127653642240, + "reserved_bytes_current_delta": 12392071168, + "reserved_bytes_peak": 127653642240, + "reserved_bytes_peak_delta": 12392071168, + "segment_count_current": 927, + "segment_count_current_delta": 9, + "segment_count_peak": 927, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190123761250Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.838410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190123761250Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190123761250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4afe09f92698d88ee0460c721618667e62c36f6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190123761250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45bc9e44f1099f98e31f0ccabcf707d6e012cad3f237050c7bd8faa45a9f7c1b +size 4528563 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190124485044Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190124485044Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dbdc057228ec25bf3f85fc5792e091e1561b0916 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190124485044Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42489992192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124978663424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42489992192, + "active_bytes_peak": 124978663424, + "allocated_bytes_current": 42489992192, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 16226069504, + "inactive_split_bytes_peak": 28322138112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127653642240, + "reserved_bytes_peak": 127653642240, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:01:24.484615Z", + "world_size": 8 + }, + "allocated_bytes_current": 42489992192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124840251392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42489992192, + "active_bytes_peak": 124978663424, + "allocated_bytes_current": 42489992192, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 16318344192, + "inactive_split_bytes_peak": 28322138112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127653642240, + "reserved_bytes_peak": 127653642240, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:01:23.848480Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16226069504, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28322138112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127653642240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127653642240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190124485044Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.559830Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190124485044Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190124485044Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4fb1949aa7a108db726c15e0b17d74bbc622c0f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190124485044Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aa16a4dbc506ba15179c45dd90f5f780091010bb916de7ebce2e9cc12d80e21 +size 4545977 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190126509895Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190126509895Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bd245f1dec291bce19ebae5842d645bc2277e9f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190126509895Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42507953152, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127975766528, + "active_bytes_peak_delta": 2997103104, + "after": { + "active_bytes_current": 42507953152, + "active_bytes_peak": 127975766528, + "allocated_bytes_current": 42507953152, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_peak": 42800894976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150841851904, + "reserved_bytes_peak": 150841851904, + "segment_count_current": 1092, + "segment_count_peak": 1092, + "timestamp_utc": "2026-08-26T19:01:26.509400Z", + "world_size": 8 + }, + "allocated_bytes_current": 42507953152, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124840251392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42489992192, + "active_bytes_peak": 124978663424, + "allocated_bytes_current": 42489992192, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 16226069504, + "inactive_split_bytes_peak": 28322138112, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127653642240, + "reserved_bytes_peak": 127653642240, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:01:24.569539Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 42800894976, + "inactive_split_bytes_peak_delta": 14478756864, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150841851904, + "reserved_bytes_current_delta": 23188209664, + "reserved_bytes_peak": 150841851904, + "reserved_bytes_peak_delta": 23188209664, + "segment_count_current": 1092, + "segment_count_current_delta": 165, + "segment_count_peak": 1092, + "segment_count_peak_delta": 165, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190126509895Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.595083Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190126509895Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190126509895Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86527f4fe3dd4d50cd0cfe7965734bf0854c48ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190126509895Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3c70d89877c7e964f422fcb8301c1058e5fcd8cfef0b25fc16d82fee260e15 +size 4762084 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190127191698Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190127191698Z.json new file mode 100644 index 0000000000000000000000000000000000000000..873d65010433f5c36c6ce49e2656c2625bcb8d9f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190127191698Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42507953152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127975766528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42507953152, + "active_bytes_peak": 127975766528, + "allocated_bytes_current": 42507953152, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_peak": 42800894976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150841851904, + "reserved_bytes_peak": 150841851904, + "segment_count_current": 1092, + "segment_count_peak": 1092, + "timestamp_utc": "2026-08-26T19:01:27.191290Z", + "world_size": 8 + }, + "allocated_bytes_current": 42507953152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124840251392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42507953152, + "active_bytes_peak": 127975766528, + "allocated_bytes_current": 42507953152, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_peak": 42800894976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150841851904, + "reserved_bytes_peak": 150841851904, + "segment_count_current": 1092, + "segment_count_peak": 1092, + "timestamp_utc": "2026-08-26T19:01:26.605768Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42800894976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150841851904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150841851904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1092, + "segment_count_current_delta": 0, + "segment_count_peak": 1092, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190127191698Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201737Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190127191698Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190127191698Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f034202eba8038cb753dbe1a679924fc470a4ee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190127191698Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2adab180ab53293d635a73909a71004490aef1295edb565b1bf7b0ccd9bf8c1 +size 403918 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190127792939Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190127792939Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82397a7d3216885837540d5c756ae8fc0bcfaf75 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190127792939Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42507953152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127975766528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42507953152, + "active_bytes_peak": 127975766528, + "allocated_bytes_current": 42507953152, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_peak": 42800894976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150841851904, + "reserved_bytes_peak": 150841851904, + "segment_count_current": 1092, + "segment_count_peak": 1092, + "timestamp_utc": "2026-08-26T19:01:27.792462Z", + "world_size": 8 + }, + "allocated_bytes_current": 42507953152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124840251392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42507953152, + "active_bytes_peak": 127975766528, + "allocated_bytes_current": 42507953152, + "allocated_bytes_peak": 124840251392, + "device": 5, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_peak": 42800894976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150841851904, + "reserved_bytes_peak": 150841851904, + "segment_count_current": 1092, + "segment_count_peak": 1092, + "timestamp_utc": "2026-08-26T19:01:27.202845Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18378660864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42800894976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150841851904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150841851904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1092, + "segment_count_current_delta": 0, + "segment_count_peak": 1092, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190127792939Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.802280Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190127792939Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190127792939Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a3b732979bdace074c04b50f466a055d7354e35 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190127792939Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:835d55156bff7d055445090450f955683242cb0310778423a70139170b03f534 +size 403918 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190123031894Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190123031894Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b0fea02de34de897198008969c66530f70f973c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190123031894Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42492150784, + "active_bytes_current_delta": 19446403584, + "active_bytes_peak": 110643482624, + "active_bytes_peak_delta": 87585152512, + "after": { + "active_bytes_current": 42492150784, + "active_bytes_peak": 110643482624, + "allocated_bytes_current": 42492150784, + "allocated_bytes_peak": 110643482624, + "device": 6, + "inactive_split_bytes_current": 16431528960, + "inactive_split_bytes_peak": 28266328576, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115402080256, + "reserved_bytes_peak": 115402080256, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:01:23.031379Z", + "world_size": 8 + }, + "allocated_bytes_current": 42492150784, + "allocated_bytes_current_delta": 19446403584, + "allocated_bytes_peak": 110643482624, + "allocated_bytes_peak_delta": 87585152512, + "before": { + "active_bytes_current": 23045747200, + "active_bytes_peak": 23058330112, + "allocated_bytes_current": 23045747200, + "allocated_bytes_peak": 23058330112, + "device": 6, + "inactive_split_bytes_current": 358469120, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25761415168, + "reserved_bytes_peak": 25761415168, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-26T19:00:42.978677Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16431528960, + "inactive_split_bytes_current_delta": 16073059840, + "inactive_split_bytes_peak": 28266328576, + "inactive_split_bytes_peak_delta": 27383093248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115402080256, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115402080256, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 925, + "segment_count_current_delta": 368, + "segment_count_peak": 925, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190123031894Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.115395Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190123031894Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190123031894Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9267da698542daab2d413ed1e12eab09940e0639 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190123031894Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb74076b699b269261240d4d55c46a7fc6d83100764e30857bbdcd0b348f3210 +size 4150089 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190123761248Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190123761248Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d8f716efdb0b63b5a2e2a3135066957e5ee216f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190123761248Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42492150784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125016855040, + "active_bytes_peak_delta": 14373372416, + "after": { + "active_bytes_current": 42492150784, + "active_bytes_peak": 125016855040, + "allocated_bytes_current": 42492150784, + "allocated_bytes_peak": 124878443008, + "device": 6, + "inactive_split_bytes_current": 16523803648, + "inactive_split_bytes_peak": 28521610240, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127798345728, + "reserved_bytes_peak": 127798345728, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:01:23.760830Z", + "world_size": 8 + }, + "allocated_bytes_current": 42492150784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124878443008, + "allocated_bytes_peak_delta": 14234960384, + "before": { + "active_bytes_current": 42492150784, + "active_bytes_peak": 110643482624, + "allocated_bytes_current": 42492150784, + "allocated_bytes_peak": 110643482624, + "device": 6, + "inactive_split_bytes_current": 16431528960, + "inactive_split_bytes_peak": 28266328576, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115402080256, + "reserved_bytes_peak": 115402080256, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:01:23.127056Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16523803648, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28521610240, + "inactive_split_bytes_peak_delta": 255281664, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127798345728, + "reserved_bytes_current_delta": 12396265472, + "reserved_bytes_peak": 127798345728, + "reserved_bytes_peak_delta": 12396265472, + "segment_count_current": 936, + "segment_count_current_delta": 11, + "segment_count_peak": 936, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190123761248Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.836658Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190123761248Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190123761248Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..192a7c132e6a6b411ee493f4c015a5b0776b0754 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190123761248Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e4c1ef98fb44d8c7934645a563ab213137ba21a96c29eae32f88b9aa17d9ff5 +size 4533456 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190124484978Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190124484978Z.json new file mode 100644 index 0000000000000000000000000000000000000000..832b5a65ce5acacc3edf6919e28b933d71f92dc7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190124484978Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42492150784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125294465536, + "active_bytes_peak_delta": 277610496, + "after": { + "active_bytes_current": 42492150784, + "active_bytes_peak": 125294465536, + "allocated_bytes_current": 42492150784, + "allocated_bytes_peak": 124879229440, + "device": 6, + "inactive_split_bytes_current": 16433626112, + "inactive_split_bytes_peak": 29894416384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127798345728, + "reserved_bytes_peak": 127798345728, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:01:24.484572Z", + "world_size": 8 + }, + "allocated_bytes_current": 42492150784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124879229440, + "allocated_bytes_peak_delta": 786432, + "before": { + "active_bytes_current": 42492150784, + "active_bytes_peak": 125016855040, + "allocated_bytes_current": 42492150784, + "allocated_bytes_peak": 124878443008, + "device": 6, + "inactive_split_bytes_current": 16523803648, + "inactive_split_bytes_peak": 28521610240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127798345728, + "reserved_bytes_peak": 127798345728, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:01:23.845674Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16433626112, + "inactive_split_bytes_current_delta": -90177536, + "inactive_split_bytes_peak": 29894416384, + "inactive_split_bytes_peak_delta": 1372806144, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127798345728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127798345728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 936, + "segment_count_current_delta": 0, + "segment_count_peak": 936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190124484978Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.558835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190124484978Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190124484978Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b941db6e0fb01353bf0adb837d9b906e10cd891 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190124484978Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0adf660daff0214d1a9ac1d396d04773c379027bcdefaf9a475b36e3b1b85f87 +size 4549922 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190126509896Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190126509896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e3427102a0d4cb7fe219bf45e62b67c8f39c04f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190126509896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42510111744, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127977925120, + "active_bytes_peak_delta": 2683459584, + "after": { + "active_bytes_current": 42510111744, + "active_bytes_peak": 127977925120, + "allocated_bytes_current": 42510111744, + "allocated_bytes_peak": 124879229440, + "device": 6, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_peak": 43008451584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 151051567104, + "reserved_bytes_peak": 151051567104, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:01:26.509343Z", + "world_size": 8 + }, + "allocated_bytes_current": 42510111744, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124879229440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42492150784, + "active_bytes_peak": 125294465536, + "allocated_bytes_current": 42492150784, + "allocated_bytes_peak": 124879229440, + "device": 6, + "inactive_split_bytes_current": 16433626112, + "inactive_split_bytes_peak": 29894416384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127798345728, + "reserved_bytes_peak": 127798345728, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:01:24.567691Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 43008451584, + "inactive_split_bytes_peak_delta": 13114035200, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 151051567104, + "reserved_bytes_current_delta": 23253221376, + "reserved_bytes_peak": 151051567104, + "reserved_bytes_peak_delta": 23253221376, + "segment_count_current": 1132, + "segment_count_current_delta": 196, + "segment_count_peak": 1132, + "segment_count_peak_delta": 196, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190126509896Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.595335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190126509896Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190126509896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d9f3e63f27e0c410b76ee06c7146c9605ec5bf8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190126509896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f21c9f6e2044b9cabf72334eb74dcd14fabb7e81ee097a4807013938cd2ca1f8 +size 4738635 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190127191674Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190127191674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab29df69356e96544cada93c3d3fb209b8cddd67 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190127191674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42510111744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127977925120, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42510111744, + "active_bytes_peak": 127977925120, + "allocated_bytes_current": 42510111744, + "allocated_bytes_peak": 124879229440, + "device": 6, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_peak": 43008451584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 151051567104, + "reserved_bytes_peak": 151051567104, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:01:27.191261Z", + "world_size": 8 + }, + "allocated_bytes_current": 42510111744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124879229440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42510111744, + "active_bytes_peak": 127977925120, + "allocated_bytes_current": 42510111744, + "allocated_bytes_peak": 124879229440, + "device": 6, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_peak": 43008451584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 151051567104, + "reserved_bytes_peak": 151051567104, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:01:26.605312Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 43008451584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 151051567104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151051567104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1132, + "segment_count_current_delta": 0, + "segment_count_peak": 1132, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190127191674Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201640Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190127191674Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190127191674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c5debb0b03c2e3d90231efead6078021a7d8e8f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190127191674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baa8e1b516db1a17e87dc093ea8d31d69bcc5baf86eb30ae9283a09ba1a54e01 +size 383466 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190127792826Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190127792826Z.json new file mode 100644 index 0000000000000000000000000000000000000000..324fe5521b43e565577aea8b50b9cd65b542d288 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190127792826Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42510111744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127977925120, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42510111744, + "active_bytes_peak": 127977925120, + "allocated_bytes_current": 42510111744, + "allocated_bytes_peak": 124879229440, + "device": 6, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_peak": 43008451584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 151051567104, + "reserved_bytes_peak": 151051567104, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:01:27.792437Z", + "world_size": 8 + }, + "allocated_bytes_current": 42510111744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124879229440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42510111744, + "active_bytes_peak": 127977925120, + "allocated_bytes_current": 42510111744, + "allocated_bytes_peak": 124879229440, + "device": 6, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_peak": 43008451584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 151051567104, + "reserved_bytes_peak": 151051567104, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:01:27.202768Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18586217472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 43008451584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 151051567104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151051567104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1132, + "segment_count_current_delta": 0, + "segment_count_peak": 1132, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190127792826Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.801902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190127792826Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190127792826Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5f91d0692df4e1928f1168ea67ad60f5f05dd1c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190127792826Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36a8a454c575da2b8cf417a3d33ba7b9edb2a12148671bf176edc5eaf24c044f +size 383466 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190123031906Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190123031906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce7a7dafef1aa4322230428651195b04f3def581 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190123031906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42495174144, + "active_bytes_current_delta": 19446868480, + "active_bytes_peak": 110662637568, + "active_bytes_peak_delta": 87601748992, + "after": { + "active_bytes_current": 42495174144, + "active_bytes_peak": 110662637568, + "allocated_bytes_current": 42495174144, + "allocated_bytes_peak": 110662637568, + "device": 7, + "inactive_split_bytes_current": 16382368256, + "inactive_split_bytes_peak": 28267021312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115423051776, + "reserved_bytes_peak": 115423051776, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:01:23.031419Z", + "world_size": 8 + }, + "allocated_bytes_current": 42495174144, + "allocated_bytes_current_delta": 19446868480, + "allocated_bytes_peak": 110662637568, + "allocated_bytes_peak_delta": 87601748992, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 7, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:00:42.978756Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16382368256, + "inactive_split_bytes_current_delta": 15879656960, + "inactive_split_bytes_peak": 28267021312, + "inactive_split_bytes_peak_delta": 27383785984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115423051776, + "reserved_bytes_current_delta": 89638567936, + "reserved_bytes_peak": 115423051776, + "reserved_bytes_peak_delta": 89638567936, + "segment_count_current": 925, + "segment_count_current_delta": 367, + "segment_count_peak": 925, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190123031906Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.113900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190123031906Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190123031906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d568398cb49b75da7bb0dee5fb32ef7d657d2fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190123031906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4451f893d7b3a63ecfd745b7230e0ed39dd133ddd23c7967c9bd9806ab4eb71f +size 4141268 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190123761282Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190123761282Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06947039c1e4707eebbf3fa937159fbd97a91dcd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190123761282Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42495174144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 124986153984, + "active_bytes_peak_delta": 14323516416, + "after": { + "active_bytes_current": 42495174144, + "active_bytes_peak": 124986153984, + "allocated_bytes_current": 42495174144, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 16474642944, + "inactive_split_bytes_peak": 28267021312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:23.760864Z", + "world_size": 8 + }, + "allocated_bytes_current": 42495174144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124847741952, + "allocated_bytes_peak_delta": 14185104384, + "before": { + "active_bytes_current": 42495174144, + "active_bytes_peak": 110662637568, + "allocated_bytes_current": 42495174144, + "allocated_bytes_peak": 110662637568, + "device": 7, + "inactive_split_bytes_current": 16382368256, + "inactive_split_bytes_peak": 28267021312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115423051776, + "reserved_bytes_peak": 115423051776, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:01:23.125662Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16474642944, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28267021312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127815122944, + "reserved_bytes_current_delta": 12392071168, + "reserved_bytes_peak": 127815122944, + "reserved_bytes_peak_delta": 12392071168, + "segment_count_current": 934, + "segment_count_current_delta": 9, + "segment_count_peak": 934, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190123761282Z.pickle", + "timestamp_utc": "2026-08-26T19:01:23.838599Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190123761282Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190123761282Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9808c493a667ac7b64b15cf4a10b9aa98f6681f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190123761282Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3340127becc649f1c7535aacac5d146162ad157a5ce483ffdf8769b7f2f6d79b +size 4526712 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190124485056Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190124485056Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6366137450bea00e89d6238b27aa7c42110f82e2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190124485056Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42495174144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125123771392, + "active_bytes_peak_delta": 137617408, + "after": { + "active_bytes_current": 42495174144, + "active_bytes_peak": 125123771392, + "allocated_bytes_current": 42495174144, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 16382368256, + "inactive_split_bytes_peak": 28267021312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:24.484647Z", + "world_size": 8 + }, + "allocated_bytes_current": 42495174144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124847741952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42495174144, + "active_bytes_peak": 124986153984, + "allocated_bytes_current": 42495174144, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 16474642944, + "inactive_split_bytes_peak": 28267021312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:23.848641Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16382368256, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28267021312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127815122944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127815122944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190124485056Z.pickle", + "timestamp_utc": "2026-08-26T19:01:24.561659Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190124485056Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190124485056Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1fc8b7197fafef322d5ae630113801562de54b7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190124485056Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb5ed83ebf21c38d5f4054c0e2328a224019a62f1fef99a20e3647291406a786 +size 4544161 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190126510036Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190126510036Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c037f4c8110c8a4c8f1a9e2c1b4cddbaa88020cc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190126510036Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42513135104, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 127980948480, + "active_bytes_peak_delta": 2857177088, + "after": { + "active_bytes_current": 42513135104, + "active_bytes_peak": 127980948480, + "allocated_bytes_current": 42513135104, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_peak": 42957193728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:26.509458Z", + "world_size": 8 + }, + "allocated_bytes_current": 42513135104, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 124847741952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42495174144, + "active_bytes_peak": 125123771392, + "allocated_bytes_current": 42495174144, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 16382368256, + "inactive_split_bytes_peak": 28267021312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127815122944, + "reserved_bytes_peak": 127815122944, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:01:24.571569Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 42957193728, + "inactive_split_bytes_peak_delta": 14690172416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 151003332608, + "reserved_bytes_current_delta": 23188209664, + "reserved_bytes_peak": 151003332608, + "reserved_bytes_peak_delta": 23188209664, + "segment_count_current": 1099, + "segment_count_current_delta": 165, + "segment_count_peak": 1099, + "segment_count_peak_delta": 165, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190126510036Z.pickle", + "timestamp_utc": "2026-08-26T19:01:26.595638Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190126510036Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190126510036Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f284bb3cd7e789a6201d5bc6e0281d4908467ac --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190126510036Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dab787256d9b4d8dfdf88929080d35a81aa20f2bdd0bf06ae9e41d2b499b904 +size 4761216 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190127191732Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190127191732Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82179368ee9eb259cf2f60ea064fbda66c2961f8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190127191732Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42513135104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127980948480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42513135104, + "active_bytes_peak": 127980948480, + "allocated_bytes_current": 42513135104, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_peak": 42957193728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:27.191324Z", + "world_size": 8 + }, + "allocated_bytes_current": 42513135104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124847741952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42513135104, + "active_bytes_peak": 127980948480, + "allocated_bytes_current": 42513135104, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_peak": 42957193728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:26.606591Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42957193728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 151003332608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151003332608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1099, + "segment_count_current_delta": 0, + "segment_count_peak": 1099, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190127191732Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.201835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190127191732Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190127191732Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9aad84b324ee4eea89a9780adf2c41f115d6c6f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190127191732Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:379de3809460626f91b4e466bd4f832229f91984b597aef95afa0ddda4b45eb7 +size 402876 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190127792947Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190127792947Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f6fbc36fa9bd1e19af39aaa39fb507826fa1568 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190127792947Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42513135104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127980948480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42513135104, + "active_bytes_peak": 127980948480, + "allocated_bytes_current": 42513135104, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_peak": 42957193728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:27.792530Z", + "world_size": 8 + }, + "allocated_bytes_current": 42513135104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 124847741952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42513135104, + "active_bytes_peak": 127980948480, + "allocated_bytes_current": 42513135104, + "allocated_bytes_peak": 124847741952, + "device": 7, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_peak": 42957193728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 151003332608, + "reserved_bytes_peak": 151003332608, + "segment_count_current": 1099, + "segment_count_peak": 1099, + "timestamp_utc": "2026-08-26T19:01:27.202937Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18534959616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 42957193728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 151003332608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 151003332608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1099, + "segment_count_current_delta": 0, + "segment_count_peak": 1099, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190127792947Z.pickle", + "timestamp_utc": "2026-08-26T19:01:27.802423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190127792947Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190127792947Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a1820332493848ddd2ec8cde564d7aff977ee95 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190127792947Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b88c15b9bf1fbadb3fe65a8cb7797d941b5889aff648753ed527fbd40fb95e3 +size 402876 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..1bb6c37b87e271f9fafbc122343b5467530d9934 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0f0ed8894bfb01f0ec10197cab8f44a56bb3c0d2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -0,0 +1,1569 @@ +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.00 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 19:00:18.089000 593666 torch/distributed/run.py:766] +W0826 19:00:18.089000 593666 torch/distributed/run.py:766] ***************************************** +W0826 19:00:18.089000 593666 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:00:18.089000 593666 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 23657 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777 + + + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 19:00:36.296047196 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:00:36.298289554 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:00:36.298480265 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:00:36.298507160 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:00:36.298939552 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:00:36.300102348 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:00:36.300125444 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:00:36.455875627 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.850 seconds +time to initialize megatron (seconds): 6.924 +[after megatron is initialized] datetime: 2026-08-26 19:00:42 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:00:42 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:00:42 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (606.90, 616.41) + train/valid/test-data-iterators-setup ..........: (60.98, 68.70) +[before the start of training step] datetime: 2026-08-26 19:00:42 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:01:23] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 40053.3 | throughput per GPU (TFLOP/s/GPU): 3.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092142E+01 | loss scale: 1.0 | grad norm: 30.613 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 40518.90576171875 | max allocated: 105514.8388671875 | reserved: 109990.0 | max reserved: 109990.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:01:23] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 730.0 | throughput per GPU (TFLOP/s/GPU): 174.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091989E+01 | loss scale: 1.0 | grad norm: 30.575 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:01:24] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 723.7 | throughput per GPU (TFLOP/s/GPU): 176.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952293E+00 | loss scale: 1.0 | grad norm: 42.235 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:01:26] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2024.5 | throughput per GPU (TFLOP/s/GPU): 63.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476737E+00 | loss scale: 1.0 | grad norm: 7.156 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:01:27] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 682.1 | throughput per GPU (TFLOP/s/GPU): 187.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.116971E+01 | loss scale: 1.0 | grad norm: 403.082 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:01:27] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 601.1 | throughput per GPU (TFLOP/s/GPU): 212.2 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.370039E+00 | loss scale: 1.0 | grad norm: 5.076 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:01:27 +[rank0]:[W826 19:01:29.110298797 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..8db09b5b4d87c0c7d177defee19514f8055449fd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,248 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 629 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 26289 +1, 26377 +2, 26223 +3, 26223 +4, 26223 +5, 26223 +6, 26355 +7, 26377 +0, 26775 +1, 26863 +2, 26709 +3, 26709 +4, 26709 +5, 26709 +6, 26841 +7, 26863 +0, 36513 +1, 36597 +2, 36447 +3, 36445 +4, 36445 +5, 36443 +6, 36579 +7, 36597 +0, 37785 +1, 37869 +2, 37719 +3, 37717 +4, 37717 +5, 37715 +6, 37851 +7, 37869 +0, 37873 +1, 37957 +2, 37807 +3, 37805 +4, 37805 +5, 37803 +6, 37939 +7, 37957 +0, 38027 +1, 38097 +2, 37977 +3, 37957 +4, 37937 +5, 37963 +6, 38095 +7, 38093 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38827 +1, 38911 +2, 38761 +3, 38761 +4, 38761 +5, 38757 +6, 38893 +7, 38911 +0, 38829 +1, 38913 +2, 38763 +3, 38763 +4, 38763 +5, 38759 +6, 38895 +7, 38913 +0, 113843 +1, 112357 +2, 113777 +3, 113777 +4, 113777 +5, 113775 +6, 112337 +7, 112357 +0, 114491 +1, 113005 +2, 114425 +3, 114425 +4, 114425 +5, 114423 +6, 112985 +7, 113005 +0, 114491 +1, 113005 +2, 114425 +3, 114425 +4, 114425 +5, 114423 +6, 112985 +7, 113005 +0, 114491 +1, 113005 +2, 114425 +3, 114425 +4, 114425 +5, 114423 +6, 112985 +7, 113005 +0, 114491 +1, 113005 +2, 114425 +3, 114425 +4, 114425 +5, 114423 +6, 112985 +7, 113005 +0, 115171 +1, 113587 +2, 115039 +3, 115015 +4, 115059 +5, 115003 +6, 113595 +7, 113585 +0, 116353 +1, 116917 +2, 116287 +3, 116287 +4, 116287 +5, 116285 +6, 116897 +7, 116917 +0, 117047 +1, 117597 +2, 116919 +3, 116915 +4, 116947 +5, 116907 +6, 117549 +7, 117543 +0, 73121 +1, 65451 +2, 63197 +3, 63207 +4, 63463 +5, 63279 +6, 63523 +7, 65475 +0, 153343 +1, 155355 +2, 153165 +3, 153161 +4, 153249 +5, 153243 +6, 153411 +7, 155443 +0, 152509 +1, 154579 +2, 152455 +3, 152861 +4, 152823 +5, 152453 +6, 153111 +7, 154943 +0, 2055 +1, 778 +2, 778 +3, 778 +4, 778 +5, 778 +6, 1034 +7, 154495 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190243018620Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190243018620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50cc0698dcb295291fcab1fb0d43b20b2ced3038 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190243018620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42501974016, + "active_bytes_current_delta": 19456408064, + "active_bytes_peak": 111954709504, + "active_bytes_peak_delta": 88896560640, + "after": { + "active_bytes_current": 42501974016, + "active_bytes_peak": 111954709504, + "allocated_bytes_current": 42501974016, + "allocated_bytes_peak": 111954709504, + "device": 0, + "inactive_split_bytes_current": 16365082624, + "inactive_split_bytes_peak": 28305942528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115332874240, + "reserved_bytes_peak": 115332874240, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:02:43.018139Z", + "world_size": 8 + }, + "allocated_bytes_current": 42501974016, + "allocated_bytes_current_delta": 19456408064, + "allocated_bytes_peak": 111954709504, + "allocated_bytes_peak_delta": 88896560640, + "before": { + "active_bytes_current": 23045565952, + "active_bytes_peak": 23058148864, + "allocated_bytes_current": 23045565952, + "allocated_bytes_peak": 23058148864, + "device": 0, + "inactive_split_bytes_current": 352358912, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 25692209152, + "reserved_bytes_peak": 25692209152, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-26T19:02:03.074379Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16365082624, + "inactive_split_bytes_current_delta": 16012723712, + "inactive_split_bytes_peak": 28305942528, + "inactive_split_bytes_peak_delta": 27422707200, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115332874240, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115332874240, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 922, + "segment_count_current_delta": 368, + "segment_count_peak": 922, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190243018620Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.099391Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190243018620Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190243018620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..124290e21a8897be2536832ecd4ae9350708b337 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190243018620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cddb3df10b1f3a9fedf2f452bc727d88408c4d9e02edabbe9367dd4b920cdc +size 4206874 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190243753229Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190243753229Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7505c80db98f304889c52f6b8825ef8856325b11 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190243753229Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42501974016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126458219520, + "active_bytes_peak_delta": 14503510016, + "after": { + "active_bytes_current": 42501974016, + "active_bytes_peak": 126458219520, + "allocated_bytes_current": 42501974016, + "allocated_bytes_peak": 126181395456, + "device": 0, + "inactive_split_bytes_current": 16457357312, + "inactive_split_bytes_peak": 30679193600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 128230359040, + "reserved_bytes_peak": 128230359040, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:02:43.752809Z", + "world_size": 8 + }, + "allocated_bytes_current": 42501974016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126181395456, + "allocated_bytes_peak_delta": 14226685952, + "before": { + "active_bytes_current": 42501974016, + "active_bytes_peak": 111954709504, + "allocated_bytes_current": 42501974016, + "allocated_bytes_peak": 111954709504, + "device": 0, + "inactive_split_bytes_current": 16365082624, + "inactive_split_bytes_peak": 28305942528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115332874240, + "reserved_bytes_peak": 115332874240, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:02:43.109810Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16457357312, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 30679193600, + "inactive_split_bytes_peak_delta": 2373251072, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 128230359040, + "reserved_bytes_current_delta": 12897484800, + "reserved_bytes_peak": 128230359040, + "reserved_bytes_peak_delta": 12897484800, + "segment_count_current": 934, + "segment_count_current_delta": 12, + "segment_count_peak": 934, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190243753229Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.825446Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190243753229Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190243753229Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..223ed014f41c8e8bc219700f32c7f750dc889530 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190243753229Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ce4feaf7386c96f87e8a687324e522e20d8ab737ef5fd44a34f6301529b1c75 +size 4594192 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190244479946Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190244479946Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18d4e3a6cf3801ad157400fd247df058fbddddd9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190244479946Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42501974016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126597868544, + "active_bytes_peak_delta": 139649024, + "after": { + "active_bytes_current": 42501974016, + "active_bytes_peak": 126597868544, + "allocated_bytes_current": 42501974016, + "allocated_bytes_peak": 126182632448, + "device": 0, + "inactive_split_bytes_current": 16457357312, + "inactive_split_bytes_peak": 30679193600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 128230359040, + "reserved_bytes_peak": 128230359040, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:02:44.479525Z", + "world_size": 8 + }, + "allocated_bytes_current": 42501974016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126182632448, + "allocated_bytes_peak_delta": 1236992, + "before": { + "active_bytes_current": 42501974016, + "active_bytes_peak": 126458219520, + "allocated_bytes_current": 42501974016, + "allocated_bytes_peak": 126181395456, + "device": 0, + "inactive_split_bytes_current": 16457357312, + "inactive_split_bytes_peak": 30679193600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 128230359040, + "reserved_bytes_peak": 128230359040, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:02:43.833278Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16457357312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 30679193600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 128230359040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128230359040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190244479946Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.556793Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190244479946Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190244479946Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14d35e4de8f509cda5b21edcf034105dfd2b3ad1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190244479946Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc79042e4b13bc5606074ba8e90d233fd81bcb946e9fb638fd04bc7b8c22e218 +size 4611631 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190246490915Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190246490915Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d681cf254c3b7cbc4572c2c0805c21097b927fa7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190246490915Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42519934976, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129305783808, + "active_bytes_peak_delta": 2707915264, + "after": { + "active_bytes_current": 42519934976, + "active_bytes_peak": 129305783808, + "allocated_bytes_current": 42519934976, + "allocated_bytes_peak": 126182632448, + "device": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_peak": 44271782400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_peak": 153234702336, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:02:46.490389Z", + "world_size": 8 + }, + "allocated_bytes_current": 42519934976, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126182632448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42501974016, + "active_bytes_peak": 126597868544, + "allocated_bytes_current": 42501974016, + "allocated_bytes_peak": 126182632448, + "device": 0, + "inactive_split_bytes_current": 16457357312, + "inactive_split_bytes_peak": 30679193600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 128230359040, + "reserved_bytes_peak": 128230359040, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-26T19:02:44.564959Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_current_delta": 2062413824, + "inactive_split_bytes_peak": 44271782400, + "inactive_split_bytes_peak_delta": 13592588800, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_current_delta": 25004343296, + "reserved_bytes_peak": 153234702336, + "reserved_bytes_peak_delta": 25004343296, + "segment_count_current": 1137, + "segment_count_current_delta": 203, + "segment_count_peak": 1137, + "segment_count_peak_delta": 203, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190246490915Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.573419Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190246490915Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190246490915Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5a5aac10f8c15028c21898e81fcd35745fc4f72 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190246490915Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d06ba15d5814e56ea2a7fcee64b7ea83142bf236c332ac7930e9e8cea193de1b +size 4772114 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190247175066Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190247175066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80bd7d2158fa30da3c335acd2711d2ce1544cc73 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190247175066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42519934976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129305783808, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42519934976, + "active_bytes_peak": 129305783808, + "allocated_bytes_current": 42519934976, + "allocated_bytes_peak": 126182632448, + "device": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_peak": 44271782400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_peak": 153234702336, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:02:47.174650Z", + "world_size": 8 + }, + "allocated_bytes_current": 42519934976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126182632448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42519934976, + "active_bytes_peak": 129305783808, + "allocated_bytes_current": 42519934976, + "allocated_bytes_peak": 126182632448, + "device": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_peak": 44271782400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_peak": 153234702336, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:02:46.582568Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44271782400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153234702336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190247175066Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.185510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190247175066Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190247175066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a8f3d42ba3cafa910428dede86376bf30c37318 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190247175066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaa65bd0972c09486cb28192b2cb0ac9863f7cb3121f496524028df60621dd60 +size 387216 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190247780729Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190247780729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31554e8d351f92c6ce83b83e2fcec0a207dff0f9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190247780729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42519934976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129305783808, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42519934976, + "active_bytes_peak": 129305783808, + "allocated_bytes_current": 42519934976, + "allocated_bytes_peak": 126182632448, + "device": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_peak": 44271782400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_peak": 153234702336, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:02:47.780341Z", + "world_size": 8 + }, + "allocated_bytes_current": 42519934976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126182632448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42519934976, + "active_bytes_peak": 129305783808, + "allocated_bytes_current": 42519934976, + "allocated_bytes_peak": 126182632448, + "device": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_peak": 44271782400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_peak": 153234702336, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:02:47.186626Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18519771136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44271782400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 153234702336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153234702336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T190247780729Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.790125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190247780729Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190247780729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..894a75ec0db98193b0e3853a21a2d8288f7b02d7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190247780729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b97d6833d845dc941ebec6d5bad59cc3ceef1de666ebc1fb46a031d97d5b55a +size 387216 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190243018586Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190243018586Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dfbfe30d36fdc19a306fff67d26d3d7422f49065 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190243018586Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505521152, + "active_bytes_current_delta": 19457215488, + "active_bytes_peak": 111966336000, + "active_bytes_peak_delta": 88905447424, + "after": { + "active_bytes_current": 42505521152, + "active_bytes_peak": 111966336000, + "allocated_bytes_current": 42505521152, + "allocated_bytes_peak": 111966336000, + "device": 1, + "inactive_split_bytes_current": 18563545088, + "inactive_split_bytes_peak": 30498231808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115922173952, + "reserved_bytes_peak": 115922173952, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:02:43.018083Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505521152, + "allocated_bytes_current_delta": 19457215488, + "allocated_bytes_peak": 111966336000, + "allocated_bytes_peak_delta": 88905447424, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 1, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:02:03.074370Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18563545088, + "inactive_split_bytes_current_delta": 18060833792, + "inactive_split_bytes_peak": 30498231808, + "inactive_split_bytes_peak_delta": 29614996480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115922173952, + "reserved_bytes_current_delta": 90137690112, + "reserved_bytes_peak": 115922173952, + "reserved_bytes_peak_delta": 90137690112, + "segment_count_current": 925, + "segment_count_current_delta": 367, + "segment_count_peak": 925, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190243018586Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.101510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190243018586Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190243018586Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..400c70d41b97560efa72bf67e884b3765289cc6d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190243018586Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:256cecf6c914cf1fc732135c1a3190f70e0fee27dc94d85073c82298db4bd6e9 +size 4145676 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190243753272Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190243753272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c5a539535a5bbc92f1f6bd986df9acae703da71 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190243753272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505521152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126461689344, + "active_bytes_peak_delta": 14495353344, + "after": { + "active_bytes_current": 42505521152, + "active_bytes_peak": 126461689344, + "allocated_bytes_current": 42505521152, + "allocated_bytes_peak": 126184865280, + "device": 1, + "inactive_split_bytes_current": 18657916928, + "inactive_split_bytes_peak": 30498231808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 130461728768, + "reserved_bytes_peak": 130461728768, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:02:43.752853Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505521152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126184865280, + "allocated_bytes_peak_delta": 14218529280, + "before": { + "active_bytes_current": 42505521152, + "active_bytes_peak": 111966336000, + "allocated_bytes_current": 42505521152, + "allocated_bytes_peak": 111966336000, + "device": 1, + "inactive_split_bytes_current": 18563545088, + "inactive_split_bytes_peak": 30498231808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115922173952, + "reserved_bytes_peak": 115922173952, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:02:43.112871Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18657916928, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 30498231808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 130461728768, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 130461728768, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 935, + "segment_count_current_delta": 10, + "segment_count_peak": 935, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190243753272Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.827835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190243753272Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190243753272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6852d07b348bfab992850bc02c2d1ab0e5bbca6e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190243753272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aaa253f34f9e7a54f6821cb34cf49fc3fcf21e02b0263a773178a34d2b51670 +size 4530228 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190244479913Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190244479913Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e4930165972fa7e9cbd5301f3a4487924b68ea0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190244479913Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505521152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126600560128, + "active_bytes_peak_delta": 138870784, + "after": { + "active_bytes_current": 42505521152, + "active_bytes_peak": 126600560128, + "allocated_bytes_current": 42505521152, + "allocated_bytes_peak": 126185324032, + "device": 1, + "inactive_split_bytes_current": 18563545088, + "inactive_split_bytes_peak": 30498231808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 130461728768, + "reserved_bytes_peak": 130461728768, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:02:44.479492Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505521152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185324032, + "allocated_bytes_peak_delta": 458752, + "before": { + "active_bytes_current": 42505521152, + "active_bytes_peak": 126461689344, + "allocated_bytes_current": 42505521152, + "allocated_bytes_peak": 126184865280, + "device": 1, + "inactive_split_bytes_current": 18657916928, + "inactive_split_bytes_peak": 30498231808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 130461728768, + "reserved_bytes_peak": 130461728768, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:02:43.836608Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18563545088, + "inactive_split_bytes_current_delta": -94371840, + "inactive_split_bytes_peak": 30498231808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 130461728768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130461728768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 935, + "segment_count_current_delta": 0, + "segment_count_peak": 935, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190244479913Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.555979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190244479913Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190244479913Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68513ab6f3c50ed85d11359a33c24b32d460ab8b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190244479913Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84069b8a9a47b1ab104098c8fc4a48fda60089bc2fa3b38fc3ae7e0c9d312cc8 +size 4548847 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190246490981Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190246490981Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0a2d0bf588aea2ac2d653d51b85f3e8e675d91e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190246490981Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42523482112, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129309330944, + "active_bytes_peak_delta": 2708770816, + "after": { + "active_bytes_current": 42523482112, + "active_bytes_peak": 129309330944, + "allocated_bytes_current": 42523482112, + "allocated_bytes_peak": 126185324032, + "device": 1, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_peak": 46463953408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 155344437248, + "reserved_bytes_peak": 155344437248, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:02:46.490412Z", + "world_size": 8 + }, + "allocated_bytes_current": 42523482112, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126185324032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42505521152, + "active_bytes_peak": 126600560128, + "allocated_bytes_current": 42505521152, + "allocated_bytes_peak": 126185324032, + "device": 1, + "inactive_split_bytes_current": 18563545088, + "inactive_split_bytes_peak": 30498231808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 130461728768, + "reserved_bytes_peak": 130461728768, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:02:44.564878Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_current_delta": 2154688512, + "inactive_split_bytes_peak": 46463953408, + "inactive_split_bytes_peak_delta": 15965721600, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 155344437248, + "reserved_bytes_current_delta": 24882708480, + "reserved_bytes_peak": 155344437248, + "reserved_bytes_peak_delta": 24882708480, + "segment_count_current": 1123, + "segment_count_current_delta": 188, + "segment_count_peak": 1123, + "segment_count_peak_delta": 188, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190246490981Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.574888Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190246490981Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190246490981Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d479bd1e75c89fccc0970ff264102f6d9f437cb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190246490981Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5354b26da11c3468c0de0df7cb2a01fb097906696130adba5369f5e5b17666f9 +size 4765565 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190247175059Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190247175059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82112996d5eb79ccdb58692be26012964df17df5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190247175059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42523482112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129309330944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42523482112, + "active_bytes_peak": 129309330944, + "allocated_bytes_current": 42523482112, + "allocated_bytes_peak": 126185324032, + "device": 1, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_peak": 46463953408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 155344437248, + "reserved_bytes_peak": 155344437248, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:02:47.174650Z", + "world_size": 8 + }, + "allocated_bytes_current": 42523482112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185324032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42523482112, + "active_bytes_peak": 129309330944, + "allocated_bytes_current": 42523482112, + "allocated_bytes_peak": 126185324032, + "device": 1, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_peak": 46463953408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 155344437248, + "reserved_bytes_peak": 155344437248, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:02:46.584644Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46463953408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 155344437248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155344437248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190247175059Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.185414Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190247175059Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190247175059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c8e4fb0028943ed1eada51160878ccd9a1d86e1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190247175059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96d27b1893f81472a2affa539b60330afebffffd97f9ffa9df58756eed082cb0 +size 407521 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190247780729Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190247780729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a705cb07cd2f4e21accf162108211cbb7ffecfdd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190247780729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42523482112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129309330944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42523482112, + "active_bytes_peak": 129309330944, + "allocated_bytes_current": 42523482112, + "allocated_bytes_peak": 126185324032, + "device": 1, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_peak": 46463953408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 155344437248, + "reserved_bytes_peak": 155344437248, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:02:47.780336Z", + "world_size": 8 + }, + "allocated_bytes_current": 42523482112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185324032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42523482112, + "active_bytes_peak": 129309330944, + "allocated_bytes_current": 42523482112, + "allocated_bytes_peak": 126185324032, + "device": 1, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_peak": 46463953408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 155344437248, + "reserved_bytes_peak": 155344437248, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:02:47.186524Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 20718233600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46463953408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 155344437248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155344437248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190247780729Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.789966Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190247780729Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190247780729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c613050a5ec90810ebf4273c1ebbed29fc8c529 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190247780729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de3b548ca6471bbdffbe117691a77ca89b21628a777613b1deb49877d767a1c4 +size 407521 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190243018580Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190243018580Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3d6f28d3ab4b9ca5e9e26562abfcba4a8aad474 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190243018580Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42504367104, + "active_bytes_current_delta": 19453497344, + "active_bytes_peak": 111964891136, + "active_bytes_peak_delta": 88901438464, + "after": { + "active_bytes_current": 42504367104, + "active_bytes_peak": 111964891136, + "allocated_bytes_current": 42504367104, + "allocated_bytes_peak": 111964891136, + "device": 2, + "inactive_split_bytes_current": 16266220544, + "inactive_split_bytes_peak": 28272629760, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:02:43.018063Z", + "world_size": 8 + }, + "allocated_bytes_current": 42504367104, + "allocated_bytes_current_delta": 19453497344, + "allocated_bytes_peak": 111964891136, + "allocated_bytes_peak_delta": 88901438464, + "before": { + "active_bytes_current": 23050869760, + "active_bytes_peak": 23063452672, + "allocated_bytes_current": 23050869760, + "allocated_bytes_peak": 23063452672, + "device": 2, + "inactive_split_bytes_current": 363832320, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:02:03.074360Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16266220544, + "inactive_split_bytes_current_delta": 15902388224, + "inactive_split_bytes_peak": 28272629760, + "inactive_split_bytes_peak_delta": 27389394432, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115263668224, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115263668224, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 919, + "segment_count_current_delta": 368, + "segment_count_peak": 919, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190243018580Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.101122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190243018580Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190243018580Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72f72e057c451281198de5a1ceca1cf60e186dd6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190243018580Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fc6cfd227798bf8855b6a6ac2bc3d5a1c5677f83648935e0f3d006a668e5368 +size 4121670 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190243753278Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190243753278Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e3c8f892f7772c1a7eda8226689406a9524c405 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190243753278Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42504367104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126315304448, + "active_bytes_peak_delta": 14350413312, + "after": { + "active_bytes_current": 42504367104, + "active_bytes_peak": 126315304448, + "allocated_bytes_current": 42504367104, + "allocated_bytes_peak": 126176892416, + "device": 2, + "inactive_split_bytes_current": 16360592384, + "inactive_split_bytes_peak": 30649799680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 128163250176, + "reserved_bytes_peak": 128163250176, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:02:43.752868Z", + "world_size": 8 + }, + "allocated_bytes_current": 42504367104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126176892416, + "allocated_bytes_peak_delta": 14212001280, + "before": { + "active_bytes_current": 42504367104, + "active_bytes_peak": 111964891136, + "allocated_bytes_current": 42504367104, + "allocated_bytes_peak": 111964891136, + "device": 2, + "inactive_split_bytes_current": 16266220544, + "inactive_split_bytes_peak": 28272629760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:02:43.112250Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16360592384, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 30649799680, + "inactive_split_bytes_peak_delta": 2377169920, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 128163250176, + "reserved_bytes_current_delta": 12899581952, + "reserved_bytes_peak": 128163250176, + "reserved_bytes_peak_delta": 12899581952, + "segment_count_current": 932, + "segment_count_current_delta": 13, + "segment_count_peak": 932, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190243753278Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.827763Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190243753278Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190243753278Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..007269da8d7e7d7546f591d866368741b2046950 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190243753278Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fc6e987a9995c34cc1ba22c92046c578412928dfe565e4bbac1c4f52b060657 +size 4504559 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190244479888Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190244479888Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c851d8ea9930595a7cb7ee745bd30079c11e72f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190244479888Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42504367104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126454052352, + "active_bytes_peak_delta": 138747904, + "after": { + "active_bytes_current": 42504367104, + "active_bytes_peak": 126454052352, + "allocated_bytes_current": 42504367104, + "allocated_bytes_peak": 126177228288, + "device": 2, + "inactive_split_bytes_current": 16268317696, + "inactive_split_bytes_peak": 30649799680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 128163250176, + "reserved_bytes_peak": 128163250176, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:02:44.479486Z", + "world_size": 8 + }, + "allocated_bytes_current": 42504367104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126177228288, + "allocated_bytes_peak_delta": 335872, + "before": { + "active_bytes_current": 42504367104, + "active_bytes_peak": 126315304448, + "allocated_bytes_current": 42504367104, + "allocated_bytes_peak": 126176892416, + "device": 2, + "inactive_split_bytes_current": 16360592384, + "inactive_split_bytes_peak": 30649799680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 128163250176, + "reserved_bytes_peak": 128163250176, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:02:43.836371Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16268317696, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 30649799680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 128163250176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128163250176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 932, + "segment_count_current_delta": 0, + "segment_count_peak": 932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190244479888Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.553017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190244479888Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190244479888Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..016e415c5fd4ca6f103c7437b6a8dd0fd8607e7b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190244479888Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ee9e21944035766b5dfebc90d276d9ef6bcce8251d78e03f3ad1405a19613a3 +size 4522481 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190246490974Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190246490974Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf4a078228438c7af1e9d666657810ae64a7b31a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190246490974Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42522328064, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129308176896, + "active_bytes_peak_delta": 2854124544, + "after": { + "active_bytes_current": 42522328064, + "active_bytes_peak": 129308176896, + "allocated_bytes_current": 42522328064, + "allocated_bytes_peak": 126177228288, + "device": 2, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_peak": 44168726016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 153048055808, + "reserved_bytes_peak": 153048055808, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:02:46.490463Z", + "world_size": 8 + }, + "allocated_bytes_current": 42522328064, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126177228288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42504367104, + "active_bytes_peak": 126454052352, + "allocated_bytes_current": 42504367104, + "allocated_bytes_peak": 126177228288, + "device": 2, + "inactive_split_bytes_current": 16268317696, + "inactive_split_bytes_peak": 30649799680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 128163250176, + "reserved_bytes_peak": 128163250176, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:02:44.561729Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 44168726016, + "inactive_split_bytes_peak_delta": 13518926336, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 153048055808, + "reserved_bytes_current_delta": 24884805632, + "reserved_bytes_peak": 153048055808, + "reserved_bytes_peak_delta": 24884805632, + "segment_count_current": 1121, + "segment_count_current_delta": 189, + "segment_count_peak": 1121, + "segment_count_peak_delta": 189, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190246490974Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.575656Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190246490974Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190246490974Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d1b395fe3f2ab32374456c5800dd228c344cfb0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190246490974Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84b2c4de93a39a9b99ff475ea3978e6b84b87f272ef7b1a6358c027b24cc41cc +size 4737589 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190247175059Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190247175059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bfa9569cb0e6b5b945ab42060b924eb4990891df --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190247175059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42522328064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129308176896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42522328064, + "active_bytes_peak": 129308176896, + "allocated_bytes_current": 42522328064, + "allocated_bytes_peak": 126177228288, + "device": 2, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_peak": 44168726016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 153048055808, + "reserved_bytes_peak": 153048055808, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:02:47.174667Z", + "world_size": 8 + }, + "allocated_bytes_current": 42522328064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126177228288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42522328064, + "active_bytes_peak": 129308176896, + "allocated_bytes_current": 42522328064, + "allocated_bytes_peak": 126177228288, + "device": 2, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_peak": 44168726016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 153048055808, + "reserved_bytes_peak": 153048055808, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:02:46.585430Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44168726016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 153048055808, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153048055808, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1121, + "segment_count_current_delta": 0, + "segment_count_peak": 1121, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190247175059Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.184628Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190247175059Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190247175059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f5783e2d43c42809e875ad75e4d77f3485369b4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190247175059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ebc4fff349e48af568f53a67fda566a34d4342deb17343e8afaa228b7be5f13 +size 381323 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190247780782Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190247780782Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83973a6528a8ed9f99c254c4d63400b7b1cbfdca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190247780782Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42522328064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129308176896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42522328064, + "active_bytes_peak": 129308176896, + "allocated_bytes_current": 42522328064, + "allocated_bytes_peak": 126177228288, + "device": 2, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_peak": 44168726016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 153048055808, + "reserved_bytes_peak": 153048055808, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:02:47.780395Z", + "world_size": 8 + }, + "allocated_bytes_current": 42522328064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126177228288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42522328064, + "active_bytes_peak": 129308176896, + "allocated_bytes_current": 42522328064, + "allocated_bytes_peak": 126177228288, + "device": 2, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_peak": 44168726016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 153048055808, + "reserved_bytes_peak": 153048055808, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:02:47.185715Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18420909056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44168726016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 153048055808, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153048055808, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1121, + "segment_count_current_delta": 0, + "segment_count_peak": 1121, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190247780782Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.789900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190247780782Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190247780782Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5fcf0206bc57c15c78f4d2bdc567d7faf9e1a124 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190247780782Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed2afd701510344bb274ff56688f3b82e28e69afa58bfffda165ff58650473ae +size 381323 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190243018575Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190243018575Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef6675332c6ac027b4188f3323640e0cfa859804 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190243018575Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42507096064, + "active_bytes_current_delta": 19458152960, + "active_bytes_peak": 111971387392, + "active_bytes_peak_delta": 88909861376, + "after": { + "active_bytes_current": 42507096064, + "active_bytes_peak": 111971387392, + "allocated_bytes_current": 42507096064, + "allocated_bytes_peak": 111971387392, + "device": 3, + "inactive_split_bytes_current": 16261394432, + "inactive_split_bytes_peak": 28334033920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:02:43.018058Z", + "world_size": 8 + }, + "allocated_bytes_current": 42507096064, + "allocated_bytes_current_delta": 19458152960, + "allocated_bytes_peak": 111971387392, + "allocated_bytes_peak_delta": 88909861376, + "before": { + "active_bytes_current": 23048943104, + "active_bytes_peak": 23061526016, + "allocated_bytes_current": 23048943104, + "allocated_bytes_peak": 23061526016, + "device": 3, + "inactive_split_bytes_current": 468519424, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:02:03.074573Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16261394432, + "inactive_split_bytes_current_delta": 15792875008, + "inactive_split_bytes_peak": 28334033920, + "inactive_split_bytes_peak_delta": 27450798592, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115263668224, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115263668224, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 919, + "segment_count_current_delta": 368, + "segment_count_peak": 919, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190243018575Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.102236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190243018575Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190243018575Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67462ebe6ec26725e37f40b786c4edf69ba3d8e1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190243018575Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b566ace7d7ccebfe8d268dd32eed4c5ec842eda6ea6268b4fe56eab664f9610 +size 4199574 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190243753276Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190243753276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0746d95e14d301df7a013119f6861026f1d9396 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190243753276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42507096064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126324168704, + "active_bytes_peak_delta": 14352781312, + "after": { + "active_bytes_current": 42507096064, + "active_bytes_peak": 126324168704, + "allocated_bytes_current": 42507096064, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 16353669120, + "inactive_split_bytes_peak": 28518985728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:43.752861Z", + "world_size": 8 + }, + "allocated_bytes_current": 42507096064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185756672, + "allocated_bytes_peak_delta": 14214369280, + "before": { + "active_bytes_current": 42507096064, + "active_bytes_peak": 111971387392, + "allocated_bytes_current": 42507096064, + "allocated_bytes_peak": 111971387392, + "device": 3, + "inactive_split_bytes_current": 16261394432, + "inactive_split_bytes_peak": 28334033920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:02:43.114234Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16353669120, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28518985728, + "inactive_split_bytes_peak_delta": 184951808, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 128161153024, + "reserved_bytes_current_delta": 12897484800, + "reserved_bytes_peak": 128161153024, + "reserved_bytes_peak_delta": 12897484800, + "segment_count_current": 931, + "segment_count_current_delta": 12, + "segment_count_peak": 931, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190243753276Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.828130Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190243753276Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190243753276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f5304ae065b905dc6ecfc19d0d1557fbe6e1622 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190243753276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ac7c1ea51f2fe7cfdd7bcd9e42a08963a94de388382da5d2a8c0f8fe561a80 +size 4587613 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190244479884Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190244479884Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8dd35d24f257acd607f17baf0ce517f53c9c59b5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190244479884Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42507096064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126461532160, + "active_bytes_peak_delta": 137363456, + "after": { + "active_bytes_current": 42507096064, + "active_bytes_peak": 126461532160, + "allocated_bytes_current": 42507096064, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 16261394432, + "inactive_split_bytes_peak": 28518985728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:44.479481Z", + "world_size": 8 + }, + "allocated_bytes_current": 42507096064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185756672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42507096064, + "active_bytes_peak": 126324168704, + "allocated_bytes_current": 42507096064, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 16353669120, + "inactive_split_bytes_peak": 28518985728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:43.837948Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16261394432, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28518985728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 128161153024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128161153024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 931, + "segment_count_current_delta": 0, + "segment_count_peak": 931, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190244479884Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.554598Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190244479884Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190244479884Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..807bd047704a29f621fd58cd7da0778d1219b28d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190244479884Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0fb9a83d761b7d8afeddef36fd2c45da44c205ada97f1fa749668d8e14a120 +size 4606935 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190246491002Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190246491002Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fd892645f80e95e2b92caabc946a9e5283d3d11 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190246491002Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42525057024, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129310905856, + "active_bytes_peak_delta": 2849373696, + "after": { + "active_bytes_current": 42525057024, + "active_bytes_peak": 129310905856, + "allocated_bytes_current": 42525057024, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_peak": 44161802752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 153043861504, + "reserved_bytes_peak": 153043861504, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:02:46.490488Z", + "world_size": 8 + }, + "allocated_bytes_current": 42525057024, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126185756672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42507096064, + "active_bytes_peak": 126461532160, + "allocated_bytes_current": 42507096064, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 16261394432, + "inactive_split_bytes_peak": 28518985728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:44.564506Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 44161802752, + "inactive_split_bytes_peak_delta": 15642817024, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 153043861504, + "reserved_bytes_current_delta": 24882708480, + "reserved_bytes_peak": 153043861504, + "reserved_bytes_peak_delta": 24882708480, + "segment_count_current": 1119, + "segment_count_current_delta": 188, + "segment_count_peak": 1119, + "segment_count_peak_delta": 188, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190246491002Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.576753Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190246491002Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190246491002Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98587a12c605614f205c1720f49ef54bee306424 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190246491002Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47cb5db13059c8a4d126d39e759fabf790a53b93743d7764385a7870b749622b +size 4821174 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190247175070Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190247175070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64f5939857c40b3e60fba3dfc312ded5c3070a3b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190247175070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42525057024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129310905856, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42525057024, + "active_bytes_peak": 129310905856, + "allocated_bytes_current": 42525057024, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_peak": 44161802752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 153043861504, + "reserved_bytes_peak": 153043861504, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:02:47.174664Z", + "world_size": 8 + }, + "allocated_bytes_current": 42525057024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185756672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42525057024, + "active_bytes_peak": 129310905856, + "allocated_bytes_current": 42525057024, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_peak": 44161802752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 153043861504, + "reserved_bytes_peak": 153043861504, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:02:46.587630Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44161802752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 153043861504, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153043861504, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1119, + "segment_count_current_delta": 0, + "segment_count_peak": 1119, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190247175070Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.185182Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190247175070Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190247175070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..240057a1d02ed9761ec3acc3f6be39221a5ed119 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190247175070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4303c316c764392130ee66606eb7ce6b5bf20efb2681cc6734fccf1f873f3aba +size 384265 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190247780745Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190247780745Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b1b17f09d233c426ac8804861fe254ee0b7eda7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190247780745Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42525057024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129310905856, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42525057024, + "active_bytes_peak": 129310905856, + "allocated_bytes_current": 42525057024, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_peak": 44161802752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 153043861504, + "reserved_bytes_peak": 153043861504, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:02:47.780359Z", + "world_size": 8 + }, + "allocated_bytes_current": 42525057024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185756672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42525057024, + "active_bytes_peak": 129310905856, + "allocated_bytes_current": 42525057024, + "allocated_bytes_peak": 126185756672, + "device": 3, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_peak": 44161802752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 153043861504, + "reserved_bytes_peak": 153043861504, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:02:47.186292Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18413985792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44161802752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 153043861504, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153043861504, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1119, + "segment_count_current_delta": 0, + "segment_count_peak": 1119, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190247780745Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.790009Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190247780745Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190247780745Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..219773839d729b8904a1abdce5e30673dff5ddcd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190247780745Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdf0320f0857d22946945eac7212399833d8b56d81a91e3feb711be913df107f +size 384265 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190243018568Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190243018568Z.json new file mode 100644 index 0000000000000000000000000000000000000000..847ac66b7f807535fa06fac5cd1ae0f3c4561b80 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190243018568Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42516636672, + "active_bytes_current_delta": 19468253184, + "active_bytes_peak": 111963356160, + "active_bytes_peak_delta": 88902389760, + "after": { + "active_bytes_current": 42516636672, + "active_bytes_peak": 111963356160, + "allocated_bytes_current": 42516636672, + "allocated_bytes_peak": 111963356160, + "device": 4, + "inactive_split_bytes_current": 16251853824, + "inactive_split_bytes_peak": 28339516416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:02:43.018022Z", + "world_size": 8 + }, + "allocated_bytes_current": 42516636672, + "allocated_bytes_current_delta": 19468253184, + "allocated_bytes_peak": 111963356160, + "allocated_bytes_peak_delta": 88902389760, + "before": { + "active_bytes_current": 23048383488, + "active_bytes_peak": 23060966400, + "allocated_bytes_current": 23048383488, + "allocated_bytes_peak": 23060966400, + "device": 4, + "inactive_split_bytes_current": 376804352, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:02:03.074385Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16251853824, + "inactive_split_bytes_current_delta": 15875049472, + "inactive_split_bytes_peak": 28339516416, + "inactive_split_bytes_peak_delta": 27456281088, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115263668224, + "reserved_bytes_current_delta": 89640665088, + "reserved_bytes_peak": 115263668224, + "reserved_bytes_peak_delta": 89640665088, + "segment_count_current": 919, + "segment_count_current_delta": 368, + "segment_count_peak": 919, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190243018568Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.099105Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190243018568Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190243018568Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffa4078682e2812290d796a0f7db0cabfff522f1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190243018568Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ce6c80b8252ed9a9209b8fc59799e0c94eceef627b3548b21ec048180aaa2e7 +size 4121030 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190243753244Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190243753244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ad7670d1ac8a6a6685e4d5591635c83bc5d1e91 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190243753244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42516636672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126462670336, + "active_bytes_peak_delta": 14499314176, + "after": { + "active_bytes_current": 42516636672, + "active_bytes_peak": 126462670336, + "allocated_bytes_current": 42516636672, + "allocated_bytes_peak": 126185846272, + "device": 4, + "inactive_split_bytes_current": 16344128512, + "inactive_split_bytes_peak": 30608694272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:43.752829Z", + "world_size": 8 + }, + "allocated_bytes_current": 42516636672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185846272, + "allocated_bytes_peak_delta": 14222490112, + "before": { + "active_bytes_current": 42516636672, + "active_bytes_peak": 111963356160, + "allocated_bytes_current": 42516636672, + "allocated_bytes_peak": 111963356160, + "device": 4, + "inactive_split_bytes_current": 16251853824, + "inactive_split_bytes_peak": 28339516416, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115263668224, + "reserved_bytes_peak": 115263668224, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:02:43.110324Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16344128512, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 30608694272, + "inactive_split_bytes_peak_delta": 2269177856, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 128161153024, + "reserved_bytes_current_delta": 12897484800, + "reserved_bytes_peak": 128161153024, + "reserved_bytes_peak_delta": 12897484800, + "segment_count_current": 931, + "segment_count_current_delta": 12, + "segment_count_peak": 931, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190243753244Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.826880Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190243753244Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190243753244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c2465593b296e3151a00c2d5adc5cef2628212e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190243753244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d31ba4cc018f359f54afc0f11358a8ce35baa640320a5a5a6303f8b5d6cc20f +size 4504303 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190244479885Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190244479885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..725e5f562df41fcc3460980cf2f5b609d0dd879b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190244479885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42516636672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126463006208, + "active_bytes_peak_delta": 335872, + "after": { + "active_bytes_current": 42516636672, + "active_bytes_peak": 126463006208, + "allocated_bytes_current": 42516636672, + "allocated_bytes_peak": 126186182144, + "device": 4, + "inactive_split_bytes_current": 16344128512, + "inactive_split_bytes_peak": 30608694272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:44.479481Z", + "world_size": 8 + }, + "allocated_bytes_current": 42516636672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126186182144, + "allocated_bytes_peak_delta": 335872, + "before": { + "active_bytes_current": 42516636672, + "active_bytes_peak": 126462670336, + "allocated_bytes_current": 42516636672, + "allocated_bytes_peak": 126185846272, + "device": 4, + "inactive_split_bytes_current": 16344128512, + "inactive_split_bytes_peak": 30608694272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:43.835813Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16344128512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 30608694272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 128161153024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128161153024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 931, + "segment_count_current_delta": 0, + "segment_count_peak": 931, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190244479885Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.552611Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190244479885Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190244479885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2baad71a5a201b659fbdf5d6c1d538f8fcb023b9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190244479885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76e72d1dccd4430a40159f03b393a7819ced20fed30d37e4a26ea0b9ab9b3887 +size 4522904 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190246490976Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190246490976Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a61ad9b7b4183353fda7661ec0a6185f6f5267da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190246490976Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534597632, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129320446464, + "active_bytes_peak_delta": 2857440256, + "after": { + "active_bytes_current": 42534597632, + "active_bytes_peak": 129320446464, + "allocated_bytes_current": 42534597632, + "allocated_bytes_peak": 126186182144, + "device": 4, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_peak": 44156456448, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 153136136192, + "reserved_bytes_peak": 153136136192, + "segment_count_current": 1120, + "segment_count_peak": 1120, + "timestamp_utc": "2026-08-26T19:02:46.490398Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534597632, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126186182144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42516636672, + "active_bytes_peak": 126463006208, + "allocated_bytes_current": 42516636672, + "allocated_bytes_peak": 126186182144, + "device": 4, + "inactive_split_bytes_current": 16344128512, + "inactive_split_bytes_peak": 30608694272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 128161153024, + "reserved_bytes_peak": 128161153024, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:02:44.561593Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 44156456448, + "inactive_split_bytes_peak_delta": 13547762176, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 153136136192, + "reserved_bytes_current_delta": 24974983168, + "reserved_bytes_peak": 153136136192, + "reserved_bytes_peak_delta": 24974983168, + "segment_count_current": 1120, + "segment_count_current_delta": 189, + "segment_count_peak": 1120, + "segment_count_peak_delta": 189, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190246490976Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.575171Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190246490976Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190246490976Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac62a0d23d58130cac5fbe3a2f97865757118bc1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190246490976Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ad7baeadf3e14020ad3fec617826d9711a2b9af37c4aeda67a0ede27c09b018 +size 4737424 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190247175057Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190247175057Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba685a91ed37ee9cdeda7532b7d9072f9613988a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190247175057Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534597632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129320446464, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42534597632, + "active_bytes_peak": 129320446464, + "allocated_bytes_current": 42534597632, + "allocated_bytes_peak": 126186182144, + "device": 4, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_peak": 44156456448, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 153136136192, + "reserved_bytes_peak": 153136136192, + "segment_count_current": 1120, + "segment_count_peak": 1120, + "timestamp_utc": "2026-08-26T19:02:47.174651Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534597632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126186182144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42534597632, + "active_bytes_peak": 129320446464, + "allocated_bytes_current": 42534597632, + "allocated_bytes_peak": 126186182144, + "device": 4, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_peak": 44156456448, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 153136136192, + "reserved_bytes_peak": 153136136192, + "segment_count_current": 1120, + "segment_count_peak": 1120, + "timestamp_utc": "2026-08-26T19:02:46.586806Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44156456448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 153136136192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153136136192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1120, + "segment_count_current_delta": 0, + "segment_count_peak": 1120, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190247175057Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.184711Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190247175057Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190247175057Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2885a72df04e5b1ee0d04fe9a01fc0b112a38c1c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190247175057Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a67190ed07cce3f9088fd5dd3b582141e271e6bc4e1c3a0117e7412f22d242 +size 380251 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190247780748Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190247780748Z.json new file mode 100644 index 0000000000000000000000000000000000000000..918f3a5768b4a62ef8b148824eb75363d7e0a3cf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190247780748Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534597632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129320446464, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42534597632, + "active_bytes_peak": 129320446464, + "allocated_bytes_current": 42534597632, + "allocated_bytes_peak": 126186182144, + "device": 4, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_peak": 44156456448, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 153136136192, + "reserved_bytes_peak": 153136136192, + "segment_count_current": 1120, + "segment_count_peak": 1120, + "timestamp_utc": "2026-08-26T19:02:47.780353Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534597632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126186182144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42534597632, + "active_bytes_peak": 129320446464, + "allocated_bytes_current": 42534597632, + "allocated_bytes_peak": 126186182144, + "device": 4, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_peak": 44156456448, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 153136136192, + "reserved_bytes_peak": 153136136192, + "segment_count_current": 1120, + "segment_count_peak": 1120, + "timestamp_utc": "2026-08-26T19:02:47.185805Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18404445184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44156456448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 153136136192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153136136192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1120, + "segment_count_current_delta": 0, + "segment_count_peak": 1120, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190247780748Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.790056Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190247780748Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190247780748Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24885267ee2768265cb5f322a8086798acaaf3b1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190247780748Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a67e60a96b8c3fd3f102eac0282c21a768d9fc7bea1c84e39e00f3636a142cf1 +size 380251 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190243018581Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190243018581Z.json new file mode 100644 index 0000000000000000000000000000000000000000..967867fe2afff86bceafe1624129d779fe5253be --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190243018581Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42511528960, + "active_bytes_current_delta": 19458345984, + "active_bytes_peak": 111971210240, + "active_bytes_peak_delta": 88905444352, + "after": { + "active_bytes_current": 42511528960, + "active_bytes_peak": 111971210240, + "allocated_bytes_current": 42511528960, + "allocated_bytes_peak": 111971210240, + "device": 5, + "inactive_split_bytes_current": 16248572928, + "inactive_split_bytes_peak": 28321703936, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115261571072, + "reserved_bytes_peak": 115261571072, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:02:43.018081Z", + "world_size": 8 + }, + "allocated_bytes_current": 42511528960, + "allocated_bytes_current_delta": 19458345984, + "allocated_bytes_peak": 111971210240, + "allocated_bytes_peak_delta": 88905444352, + "before": { + "active_bytes_current": 23053182976, + "active_bytes_peak": 23065765888, + "allocated_bytes_current": 23053182976, + "allocated_bytes_peak": 23065765888, + "device": 5, + "inactive_split_bytes_current": 462182400, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:02:03.074514Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16248572928, + "inactive_split_bytes_current_delta": 15786390528, + "inactive_split_bytes_peak": 28321703936, + "inactive_split_bytes_peak_delta": 27438468608, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115261571072, + "reserved_bytes_current_delta": 89638567936, + "reserved_bytes_peak": 115261571072, + "reserved_bytes_peak_delta": 89638567936, + "segment_count_current": 918, + "segment_count_current_delta": 367, + "segment_count_peak": 918, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190243018581Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.100163Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190243018581Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190243018581Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ace622f136b0f701d6313bbf41d14eeafe8bed72 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190243018581Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd659b6772e45e06eea567f64c66cfb931e6969e0cbd8aa930e754609143d2c +size 4143839 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190243753252Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190243753252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..efca8f64d1bf0c146e05d38fcb88f662253cd046 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190243753252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42511528960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126323735040, + "active_bytes_peak_delta": 14352524800, + "after": { + "active_bytes_current": 42511528960, + "active_bytes_peak": 126323735040, + "allocated_bytes_current": 42511528960, + "allocated_bytes_peak": 126185323008, + "device": 5, + "inactive_split_bytes_current": 16340847616, + "inactive_split_bytes_peak": 28822047744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 128152764416, + "reserved_bytes_peak": 128152764416, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:02:43.752840Z", + "world_size": 8 + }, + "allocated_bytes_current": 42511528960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185323008, + "allocated_bytes_peak_delta": 14214112768, + "before": { + "active_bytes_current": 42511528960, + "active_bytes_peak": 111971210240, + "allocated_bytes_current": 42511528960, + "allocated_bytes_peak": 111971210240, + "device": 5, + "inactive_split_bytes_current": 16248572928, + "inactive_split_bytes_peak": 28321703936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115261571072, + "reserved_bytes_peak": 115261571072, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:02:43.112227Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16340847616, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28822047744, + "inactive_split_bytes_peak_delta": 500343808, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 128152764416, + "reserved_bytes_current_delta": 12891193344, + "reserved_bytes_peak": 128152764416, + "reserved_bytes_peak_delta": 12891193344, + "segment_count_current": 927, + "segment_count_current_delta": 9, + "segment_count_peak": 927, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190243753252Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.830281Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190243753252Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190243753252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f8772118744ee793fbe752cb779747fcb924147 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190243753252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791f8658848aa5dcfd293f655f8c54f7404b9ac408f08f039c6ed6849a1006af +size 4528263 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190244479885Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190244479885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da5226fe5c2b9e1f99652e171a027b25120093a1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190244479885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42511528960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126462671360, + "active_bytes_peak_delta": 138936320, + "after": { + "active_bytes_current": 42511528960, + "active_bytes_peak": 126462671360, + "allocated_bytes_current": 42511528960, + "allocated_bytes_peak": 126185847296, + "device": 5, + "inactive_split_bytes_current": 16342944768, + "inactive_split_bytes_peak": 28822047744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 128245039104, + "reserved_bytes_peak": 128245039104, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:02:44.479477Z", + "world_size": 8 + }, + "allocated_bytes_current": 42511528960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185847296, + "allocated_bytes_peak_delta": 524288, + "before": { + "active_bytes_current": 42511528960, + "active_bytes_peak": 126323735040, + "allocated_bytes_current": 42511528960, + "allocated_bytes_peak": 126185323008, + "device": 5, + "inactive_split_bytes_current": 16340847616, + "inactive_split_bytes_peak": 28822047744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 128152764416, + "reserved_bytes_peak": 128152764416, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:02:43.840210Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16342944768, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 28822047744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 128245039104, + "reserved_bytes_current_delta": 92274688, + "reserved_bytes_peak": 128245039104, + "reserved_bytes_peak_delta": 92274688, + "segment_count_current": 928, + "segment_count_current_delta": 1, + "segment_count_peak": 928, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190244479885Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.554734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190244479885Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190244479885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ab083f43731595a5e9a7c3e479c4878a22bb9a2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190244479885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad501e898a002b8f181902e8d5ee8aa15e3a9c82216139b9a097b3ff5c1d1660 +size 4546469 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190246491077Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190246491077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a73f6c8639082b63f2c689ae7194c176703267a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190246491077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42529489920, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129315338752, + "active_bytes_peak_delta": 2852667392, + "after": { + "active_bytes_current": 42529489920, + "active_bytes_peak": 129315338752, + "allocated_bytes_current": 42529489920, + "allocated_bytes_peak": 126185847296, + "device": 5, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_peak": 44249644544, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 153129844736, + "reserved_bytes_peak": 153129844736, + "segment_count_current": 1117, + "segment_count_peak": 1117, + "timestamp_utc": "2026-08-26T19:02:46.490462Z", + "world_size": 8 + }, + "allocated_bytes_current": 42529489920, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126185847296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42511528960, + "active_bytes_peak": 126462671360, + "allocated_bytes_current": 42511528960, + "allocated_bytes_peak": 126185847296, + "device": 5, + "inactive_split_bytes_current": 16342944768, + "inactive_split_bytes_peak": 28822047744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 128245039104, + "reserved_bytes_peak": 128245039104, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:02:44.564561Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 44249644544, + "inactive_split_bytes_peak_delta": 15427596800, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 153129844736, + "reserved_bytes_current_delta": 24884805632, + "reserved_bytes_peak": 153129844736, + "reserved_bytes_peak_delta": 24884805632, + "segment_count_current": 1117, + "segment_count_current_delta": 189, + "segment_count_peak": 1117, + "segment_count_peak_delta": 189, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190246491077Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.576868Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190246491077Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190246491077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60ca2e5d5a3b185a9ed1808d11af364685ff3f2f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190246491077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0711c8d6f2fe141663fbdf70209066c7efaf4311c71cc0b488c8526a5bb8e45a +size 4763160 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190247175083Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190247175083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42ae35121f660f927eae6e7a9ef1f771bd2e283a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190247175083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42529489920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129315338752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42529489920, + "active_bytes_peak": 129315338752, + "allocated_bytes_current": 42529489920, + "allocated_bytes_peak": 126185847296, + "device": 5, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_peak": 44249644544, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 153129844736, + "reserved_bytes_peak": 153129844736, + "segment_count_current": 1117, + "segment_count_peak": 1117, + "timestamp_utc": "2026-08-26T19:02:47.174666Z", + "world_size": 8 + }, + "allocated_bytes_current": 42529489920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185847296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42529489920, + "active_bytes_peak": 129315338752, + "allocated_bytes_current": 42529489920, + "allocated_bytes_peak": 126185847296, + "device": 5, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_peak": 44249644544, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 153129844736, + "reserved_bytes_peak": 153129844736, + "segment_count_current": 1117, + "segment_count_peak": 1117, + "timestamp_utc": "2026-08-26T19:02:46.587729Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44249644544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 153129844736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153129844736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1117, + "segment_count_current_delta": 0, + "segment_count_peak": 1117, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190247175083Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.185394Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190247175083Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190247175083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90d3ca3257bd8ea99087c78b73a7198fddf554d1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190247175083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e305b6a03778de9642d79bbb101ed5008083de4b242d335fb4031b56c2b42cd +size 405189 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190247780720Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190247780720Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c5fb0a4d0210f9f948d6aaf1db493e1fd62047f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190247780720Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42529489920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129315338752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42529489920, + "active_bytes_peak": 129315338752, + "allocated_bytes_current": 42529489920, + "allocated_bytes_peak": 126185847296, + "device": 5, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_peak": 44249644544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 153129844736, + "reserved_bytes_peak": 153129844736, + "segment_count_current": 1117, + "segment_count_peak": 1117, + "timestamp_utc": "2026-08-26T19:02:47.780331Z", + "world_size": 8 + }, + "allocated_bytes_current": 42529489920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126185847296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42529489920, + "active_bytes_peak": 129315338752, + "allocated_bytes_current": 42529489920, + "allocated_bytes_peak": 126185847296, + "device": 5, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_peak": 44249644544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 153129844736, + "reserved_bytes_peak": 153129844736, + "segment_count_current": 1117, + "segment_count_peak": 1117, + "timestamp_utc": "2026-08-26T19:02:47.186507Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18403261440, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44249644544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 153129844736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153129844736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1117, + "segment_count_current_delta": 0, + "segment_count_peak": 1117, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190247780720Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.789904Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190247780720Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190247780720Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..457f915ac09f839c285e96d142e8f17c1dd28fad --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190247780720Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f1fc7feda5db30f1b06cc02225cc31898427bc03f048e2066ba7a4efd052c6f +size 405189 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190243018606Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190243018606Z.json new file mode 100644 index 0000000000000000000000000000000000000000..448be8dc3fd3f72e363cd24ed5ea879654765dca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190243018606Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496780800, + "active_bytes_current_delta": 19451033600, + "active_bytes_peak": 111951728640, + "active_bytes_peak_delta": 88893398528, + "after": { + "active_bytes_current": 42496780800, + "active_bytes_peak": 111951728640, + "allocated_bytes_current": 42496780800, + "allocated_bytes_peak": 111951728640, + "device": 6, + "inactive_split_bytes_current": 16439481856, + "inactive_split_bytes_peak": 30006334976, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115901202432, + "reserved_bytes_peak": 115901202432, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:02:43.018110Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496780800, + "allocated_bytes_current_delta": 19451033600, + "allocated_bytes_peak": 111951728640, + "allocated_bytes_peak_delta": 88893398528, + "before": { + "active_bytes_current": 23045747200, + "active_bytes_peak": 23058330112, + "allocated_bytes_current": 23045747200, + "allocated_bytes_peak": 23058330112, + "device": 6, + "inactive_split_bytes_current": 358469120, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25761415168, + "reserved_bytes_peak": 25761415168, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-26T19:02:03.074333Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16439481856, + "inactive_split_bytes_current_delta": 16081012736, + "inactive_split_bytes_peak": 30006334976, + "inactive_split_bytes_peak_delta": 29123099648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115901202432, + "reserved_bytes_current_delta": 90139787264, + "reserved_bytes_peak": 115901202432, + "reserved_bytes_peak_delta": 90139787264, + "segment_count_current": 925, + "segment_count_current_delta": 368, + "segment_count_peak": 925, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190243018606Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.106267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190243018606Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190243018606Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0ad5094aa2e8f27fd0c901c00ea69af3328121c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190243018606Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba73c2328de688526dc0c1f7beb4026729faccb761deb028e422e1030ad0471d +size 4128152 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190243753298Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190243753298Z.json new file mode 100644 index 0000000000000000000000000000000000000000..97ddf77a134d4929ebfabc79b4ce13c2e4f92a82 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190243753298Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496780800, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126326636544, + "active_bytes_peak_delta": 14374907904, + "after": { + "active_bytes_current": 42496780800, + "active_bytes_peak": 126326636544, + "allocated_bytes_current": 42496780800, + "allocated_bytes_peak": 126188224512, + "device": 6, + "inactive_split_bytes_current": 16533853696, + "inactive_split_bytes_peak": 31196091904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 130447048704, + "reserved_bytes_peak": 130447048704, + "segment_count_current": 938, + "segment_count_peak": 938, + "timestamp_utc": "2026-08-26T19:02:43.752887Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496780800, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126188224512, + "allocated_bytes_peak_delta": 14236495872, + "before": { + "active_bytes_current": 42496780800, + "active_bytes_peak": 111951728640, + "allocated_bytes_current": 42496780800, + "allocated_bytes_peak": 111951728640, + "device": 6, + "inactive_split_bytes_current": 16439481856, + "inactive_split_bytes_peak": 30006334976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115901202432, + "reserved_bytes_peak": 115901202432, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:02:43.117469Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16533853696, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 31196091904, + "inactive_split_bytes_peak_delta": 1189756928, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 130447048704, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 130447048704, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 938, + "segment_count_current_delta": 13, + "segment_count_peak": 938, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190243753298Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.828342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190243753298Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190243753298Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2a498df6e90a143a07361d7c857cc64386255f7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190243753298Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07505e5ba3f853f107833ad2bec9a53fcd44c0aa0e155f411c4d1a54193beee2 +size 4509144 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190244479927Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190244479927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3cdb678e75d1e4eec1f0fc3b86929dd28ad86c1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190244479927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496780800, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126469242880, + "active_bytes_peak_delta": 142606336, + "after": { + "active_bytes_current": 42496780800, + "active_bytes_peak": 126469242880, + "allocated_bytes_current": 42496780800, + "allocated_bytes_peak": 126192418816, + "device": 6, + "inactive_split_bytes_current": 16533853696, + "inactive_split_bytes_peak": 31196091904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 130447048704, + "reserved_bytes_peak": 130447048704, + "segment_count_current": 938, + "segment_count_peak": 938, + "timestamp_utc": "2026-08-26T19:02:44.479507Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496780800, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126192418816, + "allocated_bytes_peak_delta": 4194304, + "before": { + "active_bytes_current": 42496780800, + "active_bytes_peak": 126326636544, + "allocated_bytes_current": 42496780800, + "allocated_bytes_peak": 126188224512, + "device": 6, + "inactive_split_bytes_current": 16533853696, + "inactive_split_bytes_peak": 31196091904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 130447048704, + "reserved_bytes_peak": 130447048704, + "segment_count_current": 938, + "segment_count_peak": 938, + "timestamp_utc": "2026-08-26T19:02:43.837141Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16533853696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 31196091904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 130447048704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130447048704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 938, + "segment_count_current_delta": 0, + "segment_count_peak": 938, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190244479927Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.556104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190244479927Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190244479927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..963b500b0c00d3e73a7f0e225d4280f1503a1647 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190244479927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52ac66fa7213dc8280a2caa9f2360e30dca2e28881d7cba4913369448e5bec52 +size 4526304 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190246491026Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190246491026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94e8c7bd1a38680dffca5c28ce0a81de86a47076 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190246491026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42514741760, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129300590592, + "active_bytes_peak_delta": 2831347712, + "after": { + "active_bytes_current": 42514741760, + "active_bytes_peak": 129300590592, + "allocated_bytes_current": 42514741760, + "allocated_bytes_peak": 126192418816, + "device": 6, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_peak": 44341987328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 153306005504, + "reserved_bytes_peak": 153306005504, + "segment_count_current": 1141, + "segment_count_peak": 1141, + "timestamp_utc": "2026-08-26T19:02:46.490480Z", + "world_size": 8 + }, + "allocated_bytes_current": 42514741760, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126192418816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42496780800, + "active_bytes_peak": 126469242880, + "allocated_bytes_current": 42496780800, + "allocated_bytes_peak": 126192418816, + "device": 6, + "inactive_split_bytes_current": 16533853696, + "inactive_split_bytes_peak": 31196091904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 130447048704, + "reserved_bytes_peak": 130447048704, + "segment_count_current": 938, + "segment_count_peak": 938, + "timestamp_utc": "2026-08-26T19:02:44.565190Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 44341987328, + "inactive_split_bytes_peak_delta": 13145895424, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 153306005504, + "reserved_bytes_current_delta": 22858956800, + "reserved_bytes_peak": 153306005504, + "reserved_bytes_peak_delta": 22858956800, + "segment_count_current": 1141, + "segment_count_current_delta": 203, + "segment_count_peak": 1141, + "segment_count_peak_delta": 203, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190246491026Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.577226Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190246491026Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190246491026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58c1cd39b73a96d4755f4177a403dbef71916e47 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190246491026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71eb4b1f744cdfd9f0a38332d27c2923b5b312d2f5b1a8b64aa72361f93941e +size 4744537 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190247175110Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190247175110Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ee27917437eb87837b496d450295489d18c5e4e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190247175110Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42514741760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129300590592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42514741760, + "active_bytes_peak": 129300590592, + "allocated_bytes_current": 42514741760, + "allocated_bytes_peak": 126192418816, + "device": 6, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_peak": 44341987328, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 153306005504, + "reserved_bytes_peak": 153306005504, + "segment_count_current": 1141, + "segment_count_peak": 1141, + "timestamp_utc": "2026-08-26T19:02:47.174688Z", + "world_size": 8 + }, + "allocated_bytes_current": 42514741760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126192418816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42514741760, + "active_bytes_peak": 129300590592, + "allocated_bytes_current": 42514741760, + "allocated_bytes_peak": 126192418816, + "device": 6, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_peak": 44341987328, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 153306005504, + "reserved_bytes_peak": 153306005504, + "segment_count_current": 1141, + "segment_count_peak": 1141, + "timestamp_utc": "2026-08-26T19:02:46.587327Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44341987328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 153306005504, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153306005504, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1141, + "segment_count_current_delta": 0, + "segment_count_peak": 1141, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190247175110Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.186315Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190247175110Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190247175110Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..71fad7bf7f403c4e62efca407610386126af3cef --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190247175110Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb8ffda626b30ab1773d0deac659685b1b40da1f1894e54f77f013a36a322b72 +size 386089 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190247780751Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190247780751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e8a8cae89284f87d46198623e3d9008b3f77a579 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190247780751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42514741760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129300590592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42514741760, + "active_bytes_peak": 129300590592, + "allocated_bytes_current": 42514741760, + "allocated_bytes_peak": 126192418816, + "device": 6, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_peak": 44341987328, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 153306005504, + "reserved_bytes_peak": 153306005504, + "segment_count_current": 1141, + "segment_count_peak": 1141, + "timestamp_utc": "2026-08-26T19:02:47.780368Z", + "world_size": 8 + }, + "allocated_bytes_current": 42514741760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126192418816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42514741760, + "active_bytes_peak": 129300590592, + "allocated_bytes_current": 42514741760, + "allocated_bytes_peak": 126192418816, + "device": 6, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_peak": 44341987328, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 153306005504, + "reserved_bytes_peak": 153306005504, + "segment_count_current": 1141, + "segment_count_peak": 1141, + "timestamp_utc": "2026-08-26T19:02:47.187436Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18594170368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 44341987328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 153306005504, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 153306005504, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1141, + "segment_count_current_delta": 0, + "segment_count_peak": 1141, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190247780751Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.790401Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190247780751Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190247780751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a97ebce1e875d2ecbdb2ffad89c3aea364270b17 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190247780751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b3917f64695d45fee9a4f926266d3eb6fc56b3ecf3deb26186ae72d36e2a073 +size 386089 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190243018580Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190243018580Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3cadae4e0b3f3d98d0d6f65f5b13ee567b8c41d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190243018580Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42506421248, + "active_bytes_current_delta": 19458115584, + "active_bytes_peak": 111966336000, + "active_bytes_peak_delta": 88905447424, + "after": { + "active_bytes_current": 42506421248, + "active_bytes_peak": 111966336000, + "allocated_bytes_current": 42506421248, + "allocated_bytes_peak": 111966336000, + "device": 7, + "inactive_split_bytes_current": 18562644992, + "inactive_split_bytes_peak": 30498231808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115922173952, + "reserved_bytes_peak": 115922173952, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:02:43.018067Z", + "world_size": 8 + }, + "allocated_bytes_current": 42506421248, + "allocated_bytes_current_delta": 19458115584, + "allocated_bytes_peak": 111966336000, + "allocated_bytes_peak_delta": 88905447424, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 7, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:02:03.074503Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18562644992, + "inactive_split_bytes_current_delta": 18059933696, + "inactive_split_bytes_peak": 30498231808, + "inactive_split_bytes_peak_delta": 29614996480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115922173952, + "reserved_bytes_current_delta": 90137690112, + "reserved_bytes_peak": 115922173952, + "reserved_bytes_peak_delta": 90137690112, + "segment_count_current": 925, + "segment_count_current_delta": 367, + "segment_count_peak": 925, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190243018580Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.099719Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190243018580Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190243018580Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..657c9f14136e2cb71fc5f5274bd33444780b3e5a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190243018580Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40c65b8d2c9c6a436b4aca335daca71eb4b3a96b7dbfadc7c11712fd78df9784 +size 4141553 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190243753309Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190243753309Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60429e1ca9fd9e2bd2b5677a7d09fc7417678c69 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190243753309Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42506421248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126458726912, + "active_bytes_peak_delta": 14492390912, + "after": { + "active_bytes_current": 42506421248, + "active_bytes_peak": 126458726912, + "allocated_bytes_current": 42506421248, + "allocated_bytes_peak": 126181902848, + "device": 7, + "inactive_split_bytes_current": 18657016832, + "inactive_split_bytes_peak": 30498231808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 130461728768, + "reserved_bytes_peak": 130461728768, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:02:43.752901Z", + "world_size": 8 + }, + "allocated_bytes_current": 42506421248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126181902848, + "allocated_bytes_peak_delta": 14215566848, + "before": { + "active_bytes_current": 42506421248, + "active_bytes_peak": 111966336000, + "allocated_bytes_current": 42506421248, + "allocated_bytes_peak": 111966336000, + "device": 7, + "inactive_split_bytes_current": 18562644992, + "inactive_split_bytes_peak": 30498231808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115922173952, + "reserved_bytes_peak": 115922173952, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:02:43.111725Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18657016832, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 30498231808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 130461728768, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 130461728768, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 935, + "segment_count_current_delta": 10, + "segment_count_peak": 935, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190243753309Z.pickle", + "timestamp_utc": "2026-08-26T19:02:43.829809Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190243753309Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190243753309Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce1d82c98f84e09144c83f16b69eaf3415952839 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190243753309Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3d2fe2e8c52d4076033d5be7787652076a0d15607e162c31011321a2cd6836f +size 4527398 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190244479918Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190244479918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86d304ef19972483d784fbded52f354be98482da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190244479918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42506421248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 126460963328, + "active_bytes_peak_delta": 2236416, + "after": { + "active_bytes_current": 42506421248, + "active_bytes_peak": 126460963328, + "allocated_bytes_current": 42506421248, + "allocated_bytes_peak": 126184139264, + "device": 7, + "inactive_split_bytes_current": 18654919680, + "inactive_split_bytes_peak": 30498231808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 130554003456, + "reserved_bytes_peak": 130554003456, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:02:44.479506Z", + "world_size": 8 + }, + "allocated_bytes_current": 42506421248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126184139264, + "allocated_bytes_peak_delta": 2236416, + "before": { + "active_bytes_current": 42506421248, + "active_bytes_peak": 126458726912, + "allocated_bytes_current": 42506421248, + "allocated_bytes_peak": 126181902848, + "device": 7, + "inactive_split_bytes_current": 18657016832, + "inactive_split_bytes_peak": 30498231808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 130461728768, + "reserved_bytes_peak": 130461728768, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:02:43.839779Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18654919680, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 30498231808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 130554003456, + "reserved_bytes_current_delta": 92274688, + "reserved_bytes_peak": 130554003456, + "reserved_bytes_peak_delta": 92274688, + "segment_count_current": 936, + "segment_count_current_delta": 1, + "segment_count_peak": 936, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190244479918Z.pickle", + "timestamp_utc": "2026-08-26T19:02:44.555012Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190244479918Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190244479918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dfe0e2cc1e4c6a2dc664d883e3a8c34cef6b87f6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190244479918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82e7765e11a2116825b673ed6158af544519ff1764452f7dfc0cc9c7da6db216 +size 4546069 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190246491042Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190246491042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bca6c91ca1756a606aa94b3b35e41de7190cf19 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190246491042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42524382208, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 129310231040, + "active_bytes_peak_delta": 2849267712, + "after": { + "active_bytes_current": 42524382208, + "active_bytes_peak": 129310231040, + "allocated_bytes_current": 42524382208, + "allocated_bytes_peak": 126184139264, + "device": 7, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_peak": 46557425152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 155436711936, + "reserved_bytes_peak": 155436711936, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:02:46.490464Z", + "world_size": 8 + }, + "allocated_bytes_current": 42524382208, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 126184139264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42506421248, + "active_bytes_peak": 126460963328, + "allocated_bytes_current": 42506421248, + "allocated_bytes_peak": 126184139264, + "device": 7, + "inactive_split_bytes_current": 18654919680, + "inactive_split_bytes_peak": 30498231808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 130554003456, + "reserved_bytes_peak": 130554003456, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:02:44.564687Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_current_delta": 2062413824, + "inactive_split_bytes_peak": 46557425152, + "inactive_split_bytes_peak_delta": 16059193344, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 155436711936, + "reserved_bytes_current_delta": 24882708480, + "reserved_bytes_peak": 155436711936, + "reserved_bytes_peak_delta": 24882708480, + "segment_count_current": 1124, + "segment_count_current_delta": 188, + "segment_count_peak": 1124, + "segment_count_peak_delta": 188, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190246491042Z.pickle", + "timestamp_utc": "2026-08-26T19:02:46.576877Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190246491042Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190246491042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b20866be2a8951ece75ae96004220c05d2ea77da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190246491042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f6291d25fc40bc9411fc28cef995a57cf12b0dc5d4dc3dc6e617974db4b98cd +size 4762686 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190247175087Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190247175087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e272d80f79a4af3a906d5663b24bf261902a2b43 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190247175087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42524382208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129310231040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42524382208, + "active_bytes_peak": 129310231040, + "allocated_bytes_current": 42524382208, + "allocated_bytes_peak": 126184139264, + "device": 7, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_peak": 46557425152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 155436711936, + "reserved_bytes_peak": 155436711936, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:02:47.174671Z", + "world_size": 8 + }, + "allocated_bytes_current": 42524382208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126184139264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42524382208, + "active_bytes_peak": 129310231040, + "allocated_bytes_current": 42524382208, + "allocated_bytes_peak": 126184139264, + "device": 7, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_peak": 46557425152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 155436711936, + "reserved_bytes_peak": 155436711936, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:02:46.587686Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46557425152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 155436711936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155436711936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1124, + "segment_count_current_delta": 0, + "segment_count_peak": 1124, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190247175087Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.185272Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190247175087Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190247175087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a14e65d45d94456eec7adab76289c7614647193 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190247175087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6969614a2de21a86887ecd64f7d9b9e52e4501b8b981359046f7985fa337a9ea +size 404621 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190247780825Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190247780825Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65baed75590eccf73d741aed5c83d3a829dbb32d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190247780825Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42524382208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129310231040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42524382208, + "active_bytes_peak": 129310231040, + "allocated_bytes_current": 42524382208, + "allocated_bytes_peak": 126184139264, + "device": 7, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_peak": 46557425152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 155436711936, + "reserved_bytes_peak": 155436711936, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:02:47.780419Z", + "world_size": 8 + }, + "allocated_bytes_current": 42524382208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 126184139264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42524382208, + "active_bytes_peak": 129310231040, + "allocated_bytes_current": 42524382208, + "allocated_bytes_peak": 126184139264, + "device": 7, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_peak": 46557425152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 155436711936, + "reserved_bytes_peak": 155436711936, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:02:47.186377Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 20717333504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46557425152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 155436711936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155436711936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1124, + "segment_count_current_delta": 0, + "segment_count_peak": 1124, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190247780825Z.pickle", + "timestamp_utc": "2026-08-26T19:02:47.790178Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190247780825Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190247780825Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8292477cc009fa628deb2a6cd338e6fd14e74ea --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190247780825Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff09f9503fb72723ae97de7ad2cd5f9aff047d461d034b788e2e43aa5046da36 +size 404621 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..c2b4a48435244ace9c31b545638c9e6634b7376a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.125 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ab23eceee4ddb1db7273c039969b773a5c5996ec --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log @@ -0,0 +1,1569 @@ +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.125 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 19:01:38.302000 598000 torch/distributed/run.py:766] +W0826 19:01:38.302000 598000 torch/distributed/run.py:766] ***************************************** +W0826 19:01:38.302000 598000 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:01:38.302000 598000 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 19:01:56.254729549 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:01:56.254735424 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:01:56.254741522 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:01:56.254854417 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:01:56.254866567 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:01:56.254919286 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:01:56.254922989 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.037 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:01:56.413106516 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.970 seconds +time to initialize megatron (seconds): 7.353 +[after megatron is initialized] datetime: 2026-08-26 19:02:02 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:02:02 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:02:03 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (598.73, 610.47) + train/valid/test-data-iterators-setup ..........: (63.41, 67.49) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:02:03 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:02:43] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 39943.8 | throughput per GPU (TFLOP/s/GPU): 3.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.620 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 13.53 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 40533.041015625 | max allocated: 106768.3310546875 | reserved: 109990.0 | max reserved: 109990.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:02:43] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 735.4 | throughput per GPU (TFLOP/s/GPU): 173.5 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.566 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:02:44] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 726.6 | throughput per GPU (TFLOP/s/GPU): 175.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952471E+00 | loss scale: 1.0 | grad norm: 42.237 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:02:46] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2010.7 | throughput per GPU (TFLOP/s/GPU): 63.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476898E+00 | loss scale: 1.0 | grad norm: 7.154 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:02:47] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 684.3 | throughput per GPU (TFLOP/s/GPU): 186.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.114793E+01 | loss scale: 1.0 | grad norm: 400.648 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:02:47] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 605.7 | throughput per GPU (TFLOP/s/GPU): 210.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.368878E+00 | loss scale: 1.0 | grad norm: 5.139 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:02:47 +[rank0]:[W826 19:02:48.004083298 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..e07f428b9b87771573b4866a043ba63745611ebe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,240 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 37 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 26289 +1, 26377 +2, 26223 +3, 26223 +4, 26223 +5, 26223 +6, 26355 +7, 26377 +0, 32145 +1, 32231 +2, 32081 +3, 32079 +4, 32079 +5, 32077 +6, 32211 +7, 32231 +0, 36901 +1, 36985 +2, 36835 +3, 36833 +4, 36833 +5, 36831 +6, 36967 +7, 36985 +0, 37785 +1, 37869 +2, 37719 +3, 37717 +4, 37717 +5, 37715 +6, 37851 +7, 37869 +0, 37873 +1, 37957 +2, 37807 +3, 37805 +4, 37805 +5, 37803 +6, 37939 +7, 37957 +0, 38087 +1, 38207 +2, 38033 +3, 38039 +4, 38031 +5, 38061 +6, 38151 +7, 38195 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38829 +1, 38913 +2, 38763 +3, 38763 +4, 38763 +5, 38759 +6, 38895 +7, 38913 +0, 38829 +1, 38913 +2, 38763 +3, 38763 +4, 38763 +5, 38759 +6, 38895 +7, 38913 +0, 113939 +1, 114295 +2, 113889 +3, 113895 +4, 113887 +5, 113871 +6, 113999 +7, 114285 +0, 114491 +1, 114833 +2, 114427 +3, 114425 +4, 114425 +5, 114423 +6, 114557 +7, 114833 +0, 114491 +1, 114833 +2, 114427 +3, 114425 +4, 114425 +5, 114423 +6, 114557 +7, 114833 +0, 114491 +1, 114833 +2, 114427 +3, 114425 +4, 114425 +5, 114423 +6, 114557 +7, 114833 +0, 114491 +1, 114833 +2, 114427 +3, 114425 +4, 114425 +5, 114423 +6, 114557 +7, 114833 +0, 117755 +1, 118097 +2, 117691 +3, 117687 +4, 117687 +5, 117687 +6, 117821 +7, 118097 +0, 118403 +1, 118745 +2, 118339 +3, 118337 +4, 118337 +5, 118335 +6, 118469 +7, 118745 +0, 119065 +1, 119339 +2, 118983 +3, 118977 +4, 119015 +5, 118977 +6, 119123 +7, 119401 +0, 66303 +1, 65693 +2, 65409 +3, 65345 +4, 65441 +5, 65343 +6, 65913 +7, 65749 +0, 155315 +1, 155657 +2, 155245 +3, 155317 +4, 155229 +5, 155249 +6, 155469 +7, 155657 +0, 155313 +1, 155655 +2, 2998 +3, 155315 +4, 155227 +5, 155247 +6, 155467 +7, 155655 +0, 1075 +1, 1075 +2, 1075 +3, 1075 +4, 1395 +5, 155247 +6, 2678 +7, 1078 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190402080562Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190402080562Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d0633576b2c144bce52824266535b7c8f08374a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190402080562Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42472113152, + "active_bytes_current_delta": 19426547200, + "active_bytes_peak": 113234684928, + "active_bytes_peak_delta": 90176536064, + "after": { + "active_bytes_current": 42472113152, + "active_bytes_peak": 113234684928, + "allocated_bytes_current": 42472113152, + "allocated_bytes_peak": 113234684928, + "device": 0, + "inactive_split_bytes_current": 18552912896, + "inactive_split_bytes_peak": 30431905792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117480357888, + "reserved_bytes_peak": 117480357888, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:04:02.080089Z", + "world_size": 8 + }, + "allocated_bytes_current": 42472113152, + "allocated_bytes_current_delta": 19426547200, + "allocated_bytes_peak": 113234684928, + "allocated_bytes_peak_delta": 90176536064, + "before": { + "active_bytes_current": 23045565952, + "active_bytes_peak": 23058148864, + "allocated_bytes_current": 23045565952, + "allocated_bytes_peak": 23058148864, + "device": 0, + "inactive_split_bytes_current": 352358912, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 25692209152, + "reserved_bytes_peak": 25692209152, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-26T19:03:23.745664Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18552912896, + "inactive_split_bytes_current_delta": 18200553984, + "inactive_split_bytes_peak": 30431905792, + "inactive_split_bytes_peak_delta": 29548670464, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117480357888, + "reserved_bytes_current_delta": 91788148736, + "reserved_bytes_peak": 117480357888, + "reserved_bytes_peak_delta": 91788148736, + "segment_count_current": 923, + "segment_count_current_delta": 369, + "segment_count_peak": 923, + "segment_count_peak_delta": 369, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190402080562Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.162692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190402080562Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190402080562Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..895d32f10e2cf834d56fc2883b24959e74d09593 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190402080562Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2d8ea21e864ce77680d89fa2ff1183bcb2ccd2e1f97a79d41b17452b97845c +size 4146712 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190402816558Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190402816558Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad6cc4096547c4674319b7597e3537296e8fc5d3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190402816558Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42472113152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127734447616, + "active_bytes_peak_delta": 14499762688, + "after": { + "active_bytes_current": 42472113152, + "active_bytes_peak": 127734447616, + "allocated_bytes_current": 42472113152, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 18647284736, + "inactive_split_bytes_peak": 30431905792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 129878720512, + "reserved_bytes_peak": 129878720512, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:02.816151Z", + "world_size": 8 + }, + "allocated_bytes_current": 42472113152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127457623552, + "allocated_bytes_peak_delta": 14222938624, + "before": { + "active_bytes_current": 42472113152, + "active_bytes_peak": 113234684928, + "allocated_bytes_current": 42472113152, + "allocated_bytes_peak": 113234684928, + "device": 0, + "inactive_split_bytes_current": 18552912896, + "inactive_split_bytes_peak": 30431905792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117480357888, + "reserved_bytes_peak": 117480357888, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:04:02.174275Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18647284736, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 30431905792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 129878720512, + "reserved_bytes_current_delta": 12398362624, + "reserved_bytes_peak": 129878720512, + "reserved_bytes_peak_delta": 12398362624, + "segment_count_current": 935, + "segment_count_current_delta": 12, + "segment_count_peak": 935, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190402816558Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.891175Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190402816558Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190402816558Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91c31b1bb7dacf7c6efc657b78b4a1e8420d8492 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190402816558Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f53a3f93f0f3c3f49666338acb3fa6c20cc291bb888c276272d4e760b48d5fc1 +size 4508318 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190403545197Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190403545197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29798fd74e05308060b9e5cb6d6f65564a650bc0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190403545197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42472113152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127734447616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42472113152, + "active_bytes_peak": 127734447616, + "allocated_bytes_current": 42472113152, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 18555010048, + "inactive_split_bytes_peak": 32025500672, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 129878720512, + "reserved_bytes_peak": 129878720512, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:03.544764Z", + "world_size": 8 + }, + "allocated_bytes_current": 42472113152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127457623552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42472113152, + "active_bytes_peak": 127734447616, + "allocated_bytes_current": 42472113152, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 18647284736, + "inactive_split_bytes_peak": 30431905792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 129878720512, + "reserved_bytes_peak": 129878720512, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:02.900059Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 18555010048, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 32025500672, + "inactive_split_bytes_peak_delta": 1593594880, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 129878720512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129878720512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 935, + "segment_count_current_delta": 0, + "segment_count_peak": 935, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190403545197Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.621770Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190403545197Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190403545197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9660ed5c6f3e39199f7d61251931d75d63c5e9c1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190403545197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8cf42f42e2bc45146dfc2c575965ac5d22d921284866e7632b460d8a4244f2 +size 4527304 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190405730396Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190405730396Z.json new file mode 100644 index 0000000000000000000000000000000000000000..23f416fff60c3363115070e43e5633f586dd4009 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190405730396Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490074112, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130550434304, + "active_bytes_peak_delta": 2815986688, + "after": { + "active_bytes_current": 42490074112, + "active_bytes_peak": 130550434304, + "allocated_bytes_current": 42490074112, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_peak": 46848678912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_peak": 155302494208, + "segment_count_current": 1143, + "segment_count_peak": 1143, + "timestamp_utc": "2026-08-26T19:04:05.729865Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490074112, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127457623552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42472113152, + "active_bytes_peak": 127734447616, + "allocated_bytes_current": 42472113152, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 18555010048, + "inactive_split_bytes_peak": 32025500672, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 129878720512, + "reserved_bytes_peak": 129878720512, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:03.630922Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 46848678912, + "inactive_split_bytes_peak_delta": 14823178240, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_current_delta": 25423773696, + "reserved_bytes_peak": 155302494208, + "reserved_bytes_peak_delta": 25423773696, + "segment_count_current": 1143, + "segment_count_current_delta": 208, + "segment_count_peak": 1143, + "segment_count_peak_delta": 208, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190405730396Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.816734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190405730396Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190405730396Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5b0d309196f292b11cec2d1ff9cc83772d20ff9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190405730396Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08107ee5c15f8663b884cd991097519b4b0659c2367bc657b9e417115c0ab159 +size 4742771 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190406419082Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190406419082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2271691ebf0702b311a67f0be5d058ccb41a6524 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190406419082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490074112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130550434304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42490074112, + "active_bytes_peak": 130550434304, + "allocated_bytes_current": 42490074112, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_peak": 46848678912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_peak": 155302494208, + "segment_count_current": 1143, + "segment_count_peak": 1143, + "timestamp_utc": "2026-08-26T19:04:06.418670Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490074112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127457623552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42490074112, + "active_bytes_peak": 130550434304, + "allocated_bytes_current": 42490074112, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_peak": 46848678912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_peak": 155302494208, + "segment_count_current": 1143, + "segment_count_peak": 1143, + "timestamp_utc": "2026-08-26T19:04:05.826915Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46848678912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155302494208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1143, + "segment_count_current_delta": 0, + "segment_count_peak": 1143, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190406419082Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.429461Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190406419082Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190406419082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c921b4915d96f80d2893b8aec21843e6858c8eb9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190406419082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c8c70393acb77c48937fcc3fc98f73ac82ff1dc7bbebf926545f963b29da453 +size 385167 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190407024508Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190407024508Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb9f9c2c579a5d03d665dc692650cd92390b00d9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190407024508Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490074112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130550434304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42490074112, + "active_bytes_peak": 130550434304, + "allocated_bytes_current": 42490074112, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_peak": 46848678912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_peak": 155302494208, + "segment_count_current": 1143, + "segment_count_peak": 1143, + "timestamp_utc": "2026-08-26T19:04:07.024105Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490074112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127457623552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42490074112, + "active_bytes_peak": 130550434304, + "allocated_bytes_current": 42490074112, + "allocated_bytes_peak": 127457623552, + "device": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_peak": 46848678912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_peak": 155302494208, + "segment_count_current": 1143, + "segment_count_peak": 1143, + "timestamp_utc": "2026-08-26T19:04:06.430576Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 20707601408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46848678912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 155302494208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155302494208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1143, + "segment_count_current_delta": 0, + "segment_count_peak": 1143, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T190407024508Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.034388Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190407024508Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190407024508Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff5af8f7908b138ce0885884590652dadad513c5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190407024508Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87019c5858c96469fad0b3162dbe4495a1018a37bcd0e98cc78b5042f2ccd4fb +size 385167 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190402080885Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190402080885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..415d8fdd709d3a390d3a209b68c5386d5b50a6fb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190402080885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42479737856, + "active_bytes_current_delta": 19431432192, + "active_bytes_peak": 113246329856, + "active_bytes_peak_delta": 90185441280, + "after": { + "active_bytes_current": 42479737856, + "active_bytes_peak": 113246329856, + "allocated_bytes_current": 42479737856, + "allocated_bytes_peak": 113246329856, + "device": 1, + "inactive_split_bytes_current": 18813723648, + "inactive_split_bytes_peak": 30718459904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117838970880, + "reserved_bytes_peak": 117838970880, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:04:02.080397Z", + "world_size": 8 + }, + "allocated_bytes_current": 42479737856, + "allocated_bytes_current_delta": 19431432192, + "allocated_bytes_peak": 113246329856, + "allocated_bytes_peak_delta": 90185441280, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 1, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:03:23.745515Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18813723648, + "inactive_split_bytes_current_delta": 18311012352, + "inactive_split_bytes_peak": 30718459904, + "inactive_split_bytes_peak_delta": 29835224576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117838970880, + "reserved_bytes_current_delta": 92054487040, + "reserved_bytes_peak": 117838970880, + "reserved_bytes_peak_delta": 92054487040, + "segment_count_current": 927, + "segment_count_current_delta": 369, + "segment_count_peak": 927, + "segment_count_peak_delta": 369, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190402080885Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.168318Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190402080885Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190402080885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..631d022128fd26bde927167f02d363292274ce8a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190402080885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bed8ea1d5d90750f31843f312de1bc2744de4f02056332378c3c13624f1e5d1 +size 4135583 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190402816723Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190402816723Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb542d95869fdedce906432b65e51bf770bd300b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190402816723Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42479737856, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127585469440, + "active_bytes_peak_delta": 14339139584, + "after": { + "active_bytes_current": 42479737856, + "active_bytes_peak": 127585469440, + "allocated_bytes_current": 42479737856, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 18723546112, + "inactive_split_bytes_peak": 30746526720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 130228944896, + "reserved_bytes_peak": 130228944896, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:02.816319Z", + "world_size": 8 + }, + "allocated_bytes_current": 42479737856, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447057408, + "allocated_bytes_peak_delta": 14200727552, + "before": { + "active_bytes_current": 42479737856, + "active_bytes_peak": 113246329856, + "allocated_bytes_current": 42479737856, + "allocated_bytes_peak": 113246329856, + "device": 1, + "inactive_split_bytes_current": 18813723648, + "inactive_split_bytes_peak": 30718459904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117838970880, + "reserved_bytes_peak": 117838970880, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:04:02.179311Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18723546112, + "inactive_split_bytes_current_delta": -90177536, + "inactive_split_bytes_peak": 30746526720, + "inactive_split_bytes_peak_delta": 28066816, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 130228944896, + "reserved_bytes_current_delta": 12389974016, + "reserved_bytes_peak": 130228944896, + "reserved_bytes_peak_delta": 12389974016, + "segment_count_current": 935, + "segment_count_current_delta": 8, + "segment_count_peak": 935, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190402816723Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.888538Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190402816723Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190402816723Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b9be4c28f44c79d79b0eaf610f9909568a9c58f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190402816723Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9560d1720ca1cd1c6236495f5ddc4da35e21476048cbd93d8e0ae81d414a85b5 +size 4529516 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190403545303Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190403545303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30a56972dc937aa5dc61db62a873888e0069fec0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190403545303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42479737856, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127860147200, + "active_bytes_peak_delta": 274677760, + "after": { + "active_bytes_current": 42479737856, + "active_bytes_peak": 127860147200, + "allocated_bytes_current": 42479737856, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 18905998336, + "inactive_split_bytes_peak": 30746526720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 130228944896, + "reserved_bytes_peak": 130228944896, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:03.544875Z", + "world_size": 8 + }, + "allocated_bytes_current": 42479737856, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447057408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42479737856, + "active_bytes_peak": 127585469440, + "allocated_bytes_current": 42479737856, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 18723546112, + "inactive_split_bytes_peak": 30746526720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 130228944896, + "reserved_bytes_peak": 130228944896, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:02.896547Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 18905998336, + "inactive_split_bytes_current_delta": 182452224, + "inactive_split_bytes_peak": 30746526720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 130228944896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 130228944896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 935, + "segment_count_current_delta": 0, + "segment_count_peak": 935, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190403545303Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.619616Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190403545303Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190403545303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb82ea7807a6ac85eb70c00c92d42900a4842b13 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190403545303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26569b781415370e687d1c3853a934b461a6b535b17c5c7d6fbf9d6373a5d7e4 +size 4547666 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190405730558Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190405730558Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30966e55a5364d764a99a9e6719254d541a9c2ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190405730558Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42497698816, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130558059008, + "active_bytes_peak_delta": 2697911808, + "after": { + "active_bytes_current": 42497698816, + "active_bytes_peak": 130558059008, + "allocated_bytes_current": 42497698816, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_peak": 47107392512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:05.730005Z", + "world_size": 8 + }, + "allocated_bytes_current": 42497698816, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127447057408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42479737856, + "active_bytes_peak": 127860147200, + "allocated_bytes_current": 42479737856, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 18905998336, + "inactive_split_bytes_peak": 30746526720, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 130228944896, + "reserved_bytes_peak": 130228944896, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:03.627675Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_current_delta": 2062413824, + "inactive_split_bytes_peak": 47107392512, + "inactive_split_bytes_peak_delta": 16360865792, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 155661107200, + "reserved_bytes_current_delta": 25432162304, + "reserved_bytes_peak": 155661107200, + "reserved_bytes_peak_delta": 25432162304, + "segment_count_current": 1104, + "segment_count_current_delta": 169, + "segment_count_peak": 1104, + "segment_count_peak_delta": 169, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190405730558Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.817015Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190405730558Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190405730558Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9cd15110d8133b87765ebc161e165d27f425ac77 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190405730558Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34a6e92e61f3979bdc2619f713f97057d9a621b2898dae5739357a70e2ca4737 +size 4763752 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190406419170Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190406419170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c25b4a62d7ea990b271807f8cb4c6192664b86b6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190406419170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42497698816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130558059008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42497698816, + "active_bytes_peak": 130558059008, + "allocated_bytes_current": 42497698816, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_peak": 47107392512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:06.418753Z", + "world_size": 8 + }, + "allocated_bytes_current": 42497698816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447057408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42497698816, + "active_bytes_peak": 130558059008, + "allocated_bytes_current": 42497698816, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_peak": 47107392512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:05.828098Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 47107392512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 155661107200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155661107200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1104, + "segment_count_current_delta": 0, + "segment_count_peak": 1104, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190406419170Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.428726Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190406419170Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190406419170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8c900a994c6e4df5c02cc50c4a0799fcbb0a65af --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190406419170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:645d2c46772ebd798eeb4c4d4b3c780ec3c346670ef24bdf2457c375f8978913 +size 405281 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190407024670Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190407024670Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1810c05c4ba2937852e388309e876692fb26c71 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190407024670Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42497698816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130558059008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42497698816, + "active_bytes_peak": 130558059008, + "allocated_bytes_current": 42497698816, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_peak": 47107392512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:07.024257Z", + "world_size": 8 + }, + "allocated_bytes_current": 42497698816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447057408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42497698816, + "active_bytes_peak": 130558059008, + "allocated_bytes_current": 42497698816, + "allocated_bytes_peak": 127447057408, + "device": 1, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_peak": 47107392512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:06.429824Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 20968412160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 47107392512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 155661107200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155661107200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1104, + "segment_count_current_delta": 0, + "segment_count_peak": 1104, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190407024670Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.034713Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190407024670Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190407024670Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..789ea2bff92b2106955069be620d876f97fe3d1e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190407024670Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:343f4eddb5d3ad9e0ebb632ab9cc5748b366805698f53d55aeea50a09e0da580 +size 405281 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190402080517Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190402080517Z.json new file mode 100644 index 0000000000000000000000000000000000000000..23a3cb04125ab7bc997018b453cc019fc509b3c5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190402080517Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42472430592, + "active_bytes_current_delta": 19421560832, + "active_bytes_peak": 113228211200, + "active_bytes_peak_delta": 90164758528, + "after": { + "active_bytes_current": 42472430592, + "active_bytes_peak": 113228211200, + "allocated_bytes_current": 42472430592, + "allocated_bytes_peak": 113228211200, + "device": 2, + "inactive_split_bytes_current": 18481292288, + "inactive_split_bytes_peak": 30414126080, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117413249024, + "reserved_bytes_peak": 117413249024, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:04:02.080014Z", + "world_size": 8 + }, + "allocated_bytes_current": 42472430592, + "allocated_bytes_current_delta": 19421560832, + "allocated_bytes_peak": 113228211200, + "allocated_bytes_peak_delta": 90164758528, + "before": { + "active_bytes_current": 23050869760, + "active_bytes_peak": 23063452672, + "allocated_bytes_current": 23050869760, + "allocated_bytes_peak": 23063452672, + "device": 2, + "inactive_split_bytes_current": 363832320, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:03:23.745545Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18481292288, + "inactive_split_bytes_current_delta": 18117459968, + "inactive_split_bytes_peak": 30414126080, + "inactive_split_bytes_peak_delta": 29530890752, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117413249024, + "reserved_bytes_current_delta": 91790245888, + "reserved_bytes_peak": 117413249024, + "reserved_bytes_peak_delta": 91790245888, + "segment_count_current": 921, + "segment_count_current_delta": 370, + "segment_count_peak": 921, + "segment_count_peak_delta": 370, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190402080517Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.161180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190402080517Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190402080517Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8bdf120d79d741700fae6f07a8a159039997d5f5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190402080517Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48cbd2a78c5fb9c367e246b5b9f39ba05833d95817775c5a9d6d59eff6dd93ed +size 4112124 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190402816589Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190402816589Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e4ed53216342459ccd599ae39a6113c67cc4cb02 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190402816589Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42472430592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127724231168, + "active_bytes_peak_delta": 14496019968, + "after": { + "active_bytes_current": 42472430592, + "active_bytes_peak": 127724231168, + "allocated_bytes_current": 42472430592, + "allocated_bytes_peak": 127447407104, + "device": 2, + "inactive_split_bytes_current": 18575664128, + "inactive_split_bytes_peak": 30414126080, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 129811611648, + "reserved_bytes_peak": 129811611648, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:04:02.816190Z", + "world_size": 8 + }, + "allocated_bytes_current": 42472430592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447407104, + "allocated_bytes_peak_delta": 14219195904, + "before": { + "active_bytes_current": 42472430592, + "active_bytes_peak": 113228211200, + "allocated_bytes_current": 42472430592, + "allocated_bytes_peak": 113228211200, + "device": 2, + "inactive_split_bytes_current": 18481292288, + "inactive_split_bytes_peak": 30414126080, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117413249024, + "reserved_bytes_peak": 117413249024, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:04:02.172631Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18575664128, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 30414126080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 129811611648, + "reserved_bytes_current_delta": 12398362624, + "reserved_bytes_peak": 129811611648, + "reserved_bytes_peak_delta": 12398362624, + "segment_count_current": 933, + "segment_count_current_delta": 12, + "segment_count_peak": 933, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190402816589Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.890542Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190402816589Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190402816589Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c061d0f154f9a58e83ec09c5e950b362b7a6269f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190402816589Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed845ec527b6d8c3581149b87c9b1bd6d3f45e32496bf48fb8e5e258c13dc57 +size 4502402 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190403545173Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190403545173Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fe0b79a4928199eee91f1153c554553310ccc4a1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190403545173Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42472430592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127863699968, + "active_bytes_peak_delta": 139468800, + "after": { + "active_bytes_current": 42472430592, + "active_bytes_peak": 127863699968, + "allocated_bytes_current": 42472430592, + "allocated_bytes_peak": 127448463872, + "device": 2, + "inactive_split_bytes_current": 18481292288, + "inactive_split_bytes_peak": 30414126080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 129811611648, + "reserved_bytes_peak": 129811611648, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:04:03.544781Z", + "world_size": 8 + }, + "allocated_bytes_current": 42472430592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127448463872, + "allocated_bytes_peak_delta": 1056768, + "before": { + "active_bytes_current": 42472430592, + "active_bytes_peak": 127724231168, + "allocated_bytes_current": 42472430592, + "allocated_bytes_peak": 127447407104, + "device": 2, + "inactive_split_bytes_current": 18575664128, + "inactive_split_bytes_peak": 30414126080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 129811611648, + "reserved_bytes_peak": 129811611648, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:04:02.899269Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 18481292288, + "inactive_split_bytes_current_delta": -94371840, + "inactive_split_bytes_peak": 30414126080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 129811611648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129811611648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190403545173Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.616663Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190403545173Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190403545173Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..473cbd5e281c3d851b2fbb2418ed6aec40d0f595 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190403545173Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61086434c5bd30ad1742fad0c834f63aa5fb32df6c0f619a04b27a026a08c100 +size 4519532 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190405730495Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190405730495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16b751c1bb5667acb737ac88a2af623bf8fac24d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190405730495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490391552, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130550751744, + "active_bytes_peak_delta": 2687051776, + "after": { + "active_bytes_current": 42490391552, + "active_bytes_peak": 130550751744, + "allocated_bytes_current": 42490391552, + "allocated_bytes_peak": 127448463872, + "device": 2, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_peak": 46682686464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 155229093888, + "reserved_bytes_peak": 155229093888, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:04:05.729886Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490391552, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127448463872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42472430592, + "active_bytes_peak": 127863699968, + "allocated_bytes_current": 42472430592, + "allocated_bytes_peak": 127448463872, + "device": 2, + "inactive_split_bytes_current": 18481292288, + "inactive_split_bytes_peak": 30414126080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 129811611648, + "reserved_bytes_peak": 129811611648, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-26T19:04:03.624997Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_current_delta": 2154688512, + "inactive_split_bytes_peak": 46682686464, + "inactive_split_bytes_peak_delta": 16268560384, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 155229093888, + "reserved_bytes_current_delta": 25417482240, + "reserved_bytes_peak": 155229093888, + "reserved_bytes_peak_delta": 25417482240, + "segment_count_current": 1138, + "segment_count_current_delta": 205, + "segment_count_peak": 1138, + "segment_count_peak_delta": 205, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190405730495Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.815207Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190405730495Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190405730495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cadcca5008ccc307c1f217168b3be5b8c26f1e4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190405730495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04ead82923ccb8bd10658b24b36f5f285479512fbb7bcac052702eed8247643e +size 4736564 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190406419084Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190406419084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46607f881fad87efe4f6c8d6fabe8f95a96fcff9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190406419084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490391552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130550751744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42490391552, + "active_bytes_peak": 130550751744, + "allocated_bytes_current": 42490391552, + "allocated_bytes_peak": 127448463872, + "device": 2, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_peak": 46682686464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 155229093888, + "reserved_bytes_peak": 155229093888, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:04:06.418695Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490391552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127448463872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42490391552, + "active_bytes_peak": 130550751744, + "allocated_bytes_current": 42490391552, + "allocated_bytes_peak": 127448463872, + "device": 2, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_peak": 46682686464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 155229093888, + "reserved_bytes_peak": 155229093888, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:04:05.825160Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46682686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 155229093888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155229093888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1138, + "segment_count_current_delta": 0, + "segment_count_peak": 1138, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190406419084Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.429099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190406419084Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190406419084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a392fa2edef60660b99763eeea1604d6814753e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190406419084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bbbf046e6d3e173260f3afed8188cad2763a1145d0e5038a51d4f2e3381b620 +size 381418 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190407024563Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190407024563Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f3ddb78dd6029bea846e0d6bb6e63839b3fdb6d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190407024563Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490391552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130550751744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42490391552, + "active_bytes_peak": 130550751744, + "allocated_bytes_current": 42490391552, + "allocated_bytes_peak": 127448463872, + "device": 2, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_peak": 46682686464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 155229093888, + "reserved_bytes_peak": 155229093888, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:04:07.024118Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490391552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127448463872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42490391552, + "active_bytes_peak": 130550751744, + "allocated_bytes_current": 42490391552, + "allocated_bytes_peak": 127448463872, + "device": 2, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_peak": 46682686464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 155229093888, + "reserved_bytes_peak": 155229093888, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:04:06.430178Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 20635980800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46682686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 155229093888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155229093888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1138, + "segment_count_current_delta": 0, + "segment_count_peak": 1138, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190407024563Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.033808Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190407024563Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190407024563Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6569d1283c6566a44397bb37e10d01acf40bfe6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190407024563Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00e1fe0fd6ff9332fef6254229bae9b352def9d38241d771402a9a7c9e05a0da +size 381418 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190402080519Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190402080519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60074fa2564efad280bbbfda1d98d35f8c87e350 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190402080519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42473008640, + "active_bytes_current_delta": 19424065536, + "active_bytes_peak": 113232496640, + "active_bytes_peak_delta": 90170970624, + "after": { + "active_bytes_current": 42473008640, + "active_bytes_peak": 113232496640, + "allocated_bytes_current": 42473008640, + "allocated_bytes_peak": 113232496640, + "device": 3, + "inactive_split_bytes_current": 18461839872, + "inactive_split_bytes_peak": 30528523264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117411151872, + "reserved_bytes_peak": 117411151872, + "segment_count_current": 920, + "segment_count_peak": 920, + "timestamp_utc": "2026-08-26T19:04:02.080014Z", + "world_size": 8 + }, + "allocated_bytes_current": 42473008640, + "allocated_bytes_current_delta": 19424065536, + "allocated_bytes_peak": 113232496640, + "allocated_bytes_peak_delta": 90170970624, + "before": { + "active_bytes_current": 23048943104, + "active_bytes_peak": 23061526016, + "allocated_bytes_current": 23048943104, + "allocated_bytes_peak": 23061526016, + "device": 3, + "inactive_split_bytes_current": 468519424, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:03:23.745499Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18461839872, + "inactive_split_bytes_current_delta": 17993320448, + "inactive_split_bytes_peak": 30528523264, + "inactive_split_bytes_peak_delta": 29645287936, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117411151872, + "reserved_bytes_current_delta": 91788148736, + "reserved_bytes_peak": 117411151872, + "reserved_bytes_peak_delta": 91788148736, + "segment_count_current": 920, + "segment_count_current_delta": 369, + "segment_count_peak": 920, + "segment_count_peak_delta": 369, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190402080519Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.166898Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190402080519Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190402080519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9a09c852c4a63015b9366520e9d1a8aeaacd35d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190402080519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bffc132dd4892a0cf6f66b357d5c2fd44591f686439efc74378584fc00c91bd +size 4106276 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190402816671Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190402816671Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f989d1d5d6b3f01b32633b782fa31ea616a52c0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190402816671Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42473008640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127578424832, + "active_bytes_peak_delta": 14345928192, + "after": { + "active_bytes_current": 42473008640, + "active_bytes_peak": 127578424832, + "allocated_bytes_current": 42473008640, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 18556211712, + "inactive_split_bytes_peak": 30528523264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 131954900992, + "reserved_bytes_peak": 131954900992, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:04:02.816259Z", + "world_size": 8 + }, + "allocated_bytes_current": 42473008640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127440012800, + "allocated_bytes_peak_delta": 14207516160, + "before": { + "active_bytes_current": 42473008640, + "active_bytes_peak": 113232496640, + "allocated_bytes_current": 42473008640, + "allocated_bytes_peak": 113232496640, + "device": 3, + "inactive_split_bytes_current": 18461839872, + "inactive_split_bytes_peak": 30528523264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117411151872, + "reserved_bytes_peak": 117411151872, + "segment_count_current": 920, + "segment_count_peak": 920, + "timestamp_utc": "2026-08-26T19:04:02.178805Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18556211712, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 30528523264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 131954900992, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 131954900992, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 932, + "segment_count_current_delta": 12, + "segment_count_peak": 932, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190402816671Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.892124Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190402816671Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190402816671Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bde14985d6e539e1fdc30ef38ac7e3dfb8c5333c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190402816671Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c48527fa3c3aff775bdfb70ccc8c371a97c4b5f6a27926695317c8f29711c58d +size 4499432 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190403545166Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190403545166Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6437232d481910b5421d86e1a1b9d43518defdd1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190403545166Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42473008640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127712708096, + "active_bytes_peak_delta": 134283264, + "after": { + "active_bytes_current": 42473008640, + "active_bytes_peak": 127712708096, + "allocated_bytes_current": 42473008640, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 18556211712, + "inactive_split_bytes_peak": 30528523264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 131954900992, + "reserved_bytes_peak": 131954900992, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:04:03.544778Z", + "world_size": 8 + }, + "allocated_bytes_current": 42473008640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127440012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42473008640, + "active_bytes_peak": 127578424832, + "allocated_bytes_current": 42473008640, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 18556211712, + "inactive_split_bytes_peak": 30528523264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 131954900992, + "reserved_bytes_peak": 131954900992, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:04:02.901438Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 18556211712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 30528523264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 131954900992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131954900992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 932, + "segment_count_current_delta": 0, + "segment_count_peak": 932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190403545166Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.619278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190403545166Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190403545166Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7cdde999eb940499c3fd7d626dc7912e144d6a66 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190403545166Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd2915610c05f44bebf78c4232de4f3e0bd5015ebf2552a8f0bbcbce259a2ca3 +size 4516737 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190405730494Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190405730494Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7acfe33a271e2a98c13637ad902e1889d1133e99 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190405730494Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490969600, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130551329792, + "active_bytes_peak_delta": 2838621696, + "after": { + "active_bytes_current": 42490969600, + "active_bytes_peak": 130551329792, + "allocated_bytes_current": 42490969600, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_peak": 46847783424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 155304591360, + "reserved_bytes_peak": 155304591360, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:04:05.729904Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490969600, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127440012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42473008640, + "active_bytes_peak": 127712708096, + "allocated_bytes_current": 42473008640, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 18556211712, + "inactive_split_bytes_peak": 30528523264, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 131954900992, + "reserved_bytes_peak": 131954900992, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-26T19:04:03.628342Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 46847783424, + "inactive_split_bytes_peak_delta": 16319260160, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 155304591360, + "reserved_bytes_current_delta": 23349690368, + "reserved_bytes_peak": 155304591360, + "reserved_bytes_peak_delta": 23349690368, + "segment_count_current": 1131, + "segment_count_current_delta": 199, + "segment_count_peak": 1131, + "segment_count_peak_delta": 199, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190405730494Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.817258Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190405730494Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190405730494Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95cb7870a1381c816cb9eef09cd31e7816bbdf20 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190405730494Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cdc609d7d98f13988a4c32b5e3f1a86238f76ea976ef429ff29cc54f20f5879 +size 4733040 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190406419081Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190406419081Z.json new file mode 100644 index 0000000000000000000000000000000000000000..97e53f023dc2aadf4c4e074a2f767ea308dd05ba --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190406419081Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490969600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130551329792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42490969600, + "active_bytes_peak": 130551329792, + "allocated_bytes_current": 42490969600, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_peak": 46847783424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 155304591360, + "reserved_bytes_peak": 155304591360, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:04:06.418688Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490969600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127440012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42490969600, + "active_bytes_peak": 130551329792, + "allocated_bytes_current": 42490969600, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_peak": 46847783424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 155304591360, + "reserved_bytes_peak": 155304591360, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:04:05.827702Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46847783424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 155304591360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155304591360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1131, + "segment_count_current_delta": 0, + "segment_count_peak": 1131, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190406419081Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.429412Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190406419081Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190406419081Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d432e5450105fcc40f801b09c989cf33c59e2fe9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190406419081Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:212f330f4f90fbe1fecccb46b01b9099de00115997255a7128a8613e706f109e +size 377637 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190407024491Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190407024491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0346a1a0136e08206ef8afb1f24e90f07d555ec8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190407024491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42490969600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130551329792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42490969600, + "active_bytes_peak": 130551329792, + "allocated_bytes_current": 42490969600, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_peak": 46847783424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 155304591360, + "reserved_bytes_peak": 155304591360, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:04:07.024106Z", + "world_size": 8 + }, + "allocated_bytes_current": 42490969600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127440012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42490969600, + "active_bytes_peak": 130551329792, + "allocated_bytes_current": 42490969600, + "allocated_bytes_peak": 127440012800, + "device": 3, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_peak": 46847783424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 155304591360, + "reserved_bytes_peak": 155304591360, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:04:06.430497Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 20616528384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46847783424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 155304591360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155304591360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1131, + "segment_count_current_delta": 0, + "segment_count_peak": 1131, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190407024491Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.033840Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190407024491Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190407024491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7993b8f1f7b79c0f38baf306af259c7f5b596326 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190407024491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7564a94fe49ce603a03b4244e4eeb3f2b86a58f02b5c4e83557b4d389d3f0f9 +size 377637