diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bcb58583daf7b9a7012da649c47b6c2e55e4a722 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e1ecd78195dd2b51e2dcd4598c8defa74ed9b7ae05ef2be9f31a36339e085ba +size 9942271 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcb2066b1a4a0137957e72a6174393e3610a2801 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84649044992, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129025703424, + "active_bytes_peak_delta": 6617244160, + "after": { + "active_bytes_current": 84649044992, + "active_bytes_peak": 129025703424, + "allocated_bytes_current": 84649044992, + "allocated_bytes_peak": 122408459264, + "device": 1, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_peak": 13272840192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142709096448, + "reserved_bytes_peak": 142709096448, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.321469Z", + "world_size": 8 + }, + "allocated_bytes_current": 84649044992, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122408459264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84632001536, + "active_bytes_peak": 122408459264, + "allocated_bytes_current": 84632001536, + "allocated_bytes_peak": 122408459264, + "device": 1, + "inactive_split_bytes_current": 4173997056, + "inactive_split_bytes_peak": 12200414208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 123394326528, + "reserved_bytes_peak": 123394326528, + "segment_count_current": 2923, + "segment_count_peak": 2923, + "timestamp_utc": "2026-08-26T18:57:17.862485Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_current_delta": 35385344, + "inactive_split_bytes_peak": 13272840192, + "inactive_split_bytes_peak_delta": 1072425984, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142709096448, + "reserved_bytes_current_delta": 19314769920, + "reserved_bytes_peak": 142709096448, + "reserved_bytes_peak_delta": 19314769920, + "segment_count_current": 3356, + "segment_count_current_delta": 433, + "segment_count_peak": 3356, + "segment_count_peak_delta": 433, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.536837Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51ebab03d2fed5d2067f918dd7574a688850b355 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd3ad7429d7eeaad8c56aa12c5c1898f9108ed6966ff5f03b7d59c26d6891993 +size 13044981 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b831e7497b6cf05f5680a679043cecd6a0bd969f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84649044992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129025703424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84649044992, + "active_bytes_peak": 129025703424, + "allocated_bytes_current": 84649044992, + "allocated_bytes_peak": 122408459264, + "device": 1, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_peak": 13272840192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142709096448, + "reserved_bytes_peak": 142709096448, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.966678Z", + "world_size": 8 + }, + "allocated_bytes_current": 84649044992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122408459264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84649044992, + "active_bytes_peak": 129025703424, + "allocated_bytes_current": 84649044992, + "allocated_bytes_peak": 122408459264, + "device": 1, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_peak": 13272840192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142709096448, + "reserved_bytes_peak": 142709096448, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.565839Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13272840192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142709096448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142709096448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3356, + "segment_count_current_delta": 0, + "segment_count_peak": 3356, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.991948Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eaa3f4ecac5f1eed98a2750dfb371e3bfb319e4a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11bace237037948288f21ac0d282aa1371521ae5a556b3a40f8884d69666e01d +size 1147818 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a04f8fb027e2e6a99b1886a20021147501fc390 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84649044992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129025703424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84649044992, + "active_bytes_peak": 129025703424, + "allocated_bytes_current": 84649044992, + "allocated_bytes_peak": 122408459264, + "device": 1, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_peak": 13272840192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142709096448, + "reserved_bytes_peak": 142709096448, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:22.392483Z", + "world_size": 8 + }, + "allocated_bytes_current": 84649044992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122408459264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84649044992, + "active_bytes_peak": 129025703424, + "allocated_bytes_current": 84649044992, + "allocated_bytes_peak": 122408459264, + "device": 1, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_peak": 13272840192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142709096448, + "reserved_bytes_peak": 142709096448, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.993666Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4209382400, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13272840192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142709096448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142709096448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3356, + "segment_count_current_delta": 0, + "segment_count_peak": 3356, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.416360Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e17cf65bb9df155b7e5a6a4edea79e515d31c3f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d60f38ec3abfba66f7d751f65e7bb4c0db57b61be034f1c8dda382a731d4b0 +size 1147818 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..77e8646a98935c725e9654919cb105f2f74b2c9d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 41069773824, + "active_bytes_peak": 91617038336, + "active_bytes_peak_delta": 48057203712, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 91617038336, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 91617038336, + "device": 2, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_peak": 12182193664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93296001024, + "reserved_bytes_peak": 93296001024, + "segment_count_current": 2187, + "segment_count_peak": 2187, + "timestamp_utc": "2026-08-26T18:57:14.715878Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 41069773824, + "allocated_bytes_peak": 91617038336, + "allocated_bytes_peak_delta": 48057203712, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 2, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:56:36.124259Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_current_delta": 3056401408, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 10569281024, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93296001024, + "reserved_bytes_current_delta": 48460988416, + "reserved_bytes_peak": 93296001024, + "reserved_bytes_peak_delta": 48460988416, + "segment_count_current": 2187, + "segment_count_current_delta": 1304, + "segment_count_peak": 2187, + "segment_count_peak_delta": 1304, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.902087Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96a7decdbf236858a293e1abf89e1c67b01eae2c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b37c5ff79eea33f8e00c37e16a0c30121fae013bda203537694c76d3dd26ef1d +size 9776530 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.json new file mode 100644 index 0000000000000000000000000000000000000000..faf47813f283512932f589aeb8611a71ca2b600f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122401423360, + "active_bytes_peak_delta": 30784385024, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122401423360, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122401423360, + "device": 2, + "inactive_split_bytes_current": 4214138880, + "inactive_split_bytes_peak": 12182193664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:16.268085Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122401423360, + "allocated_bytes_peak_delta": 30784385024, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 91617038336, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 91617038336, + "device": 2, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_peak": 12182193664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93296001024, + "reserved_bytes_peak": 93296001024, + "segment_count_current": 2187, + "segment_count_peak": 2187, + "timestamp_utc": "2026-08-26T18:57:14.936634Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4214138880, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 124092678144, + "reserved_bytes_current_delta": 30796677120, + "reserved_bytes_peak": 124092678144, + "reserved_bytes_peak_delta": 30796677120, + "segment_count_current": 2896, + "segment_count_current_delta": 709, + "segment_count_peak": 2896, + "segment_count_peak_delta": 709, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.452869Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e32902871dc06391703e9700a6b526a57ad6b9d7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0ed47c8b35e9481a7efdee5e8b9467741a891e79a9557961f971e65adc8030b +size 9503132 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..75970def980948987fcd11095499b887e20e0767 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84622792704, + "active_bytes_current_delta": -524288, + "active_bytes_peak": 122862320640, + "active_bytes_peak_delta": 460897280, + "after": { + "active_bytes_current": 84622792704, + "active_bytes_peak": 122862320640, + "allocated_bytes_current": 84622792704, + "allocated_bytes_peak": 122409335808, + "device": 2, + "inactive_split_bytes_current": 4418086912, + "inactive_split_bytes_peak": 12182193664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:17.636808Z", + "world_size": 8 + }, + "allocated_bytes_current": 84622792704, + "allocated_bytes_current_delta": -524288, + "allocated_bytes_peak": 122409335808, + "allocated_bytes_peak_delta": 7912448, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122401423360, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122401423360, + "device": 2, + "inactive_split_bytes_current": 4214138880, + "inactive_split_bytes_peak": 12182193664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:16.477699Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4418086912, + "inactive_split_bytes_current_delta": 203948032, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 124092678144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 124092678144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2896, + "segment_count_current_delta": 0, + "segment_count_peak": 2896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.824121Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..465da59d7c7ec576d6022e8844583d2d5d33c331 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc955e4105c6e2315399dc8ef2b4a81962c2f9b2d7105e2abfd4462d12b09629 +size 9479267 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d036f4e6fc0251f9076b9baa18af65aee92cf44 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 17567744, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 6154698240, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409335808, + "device": 2, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.321537Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 17567744, + "allocated_bytes_peak": 122409335808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84622792704, + "active_bytes_peak": 122862320640, + "allocated_bytes_current": 84622792704, + "allocated_bytes_peak": 122409335808, + "device": 2, + "inactive_split_bytes_current": 4418086912, + "inactive_split_bytes_peak": 12182193664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:17.848657Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": -67899392, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 1132885504, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142943977472, + "reserved_bytes_current_delta": 18851299328, + "reserved_bytes_peak": 142943977472, + "reserved_bytes_peak_delta": 18851299328, + "segment_count_current": 3336, + "segment_count_current_delta": 440, + "segment_count_peak": 3336, + "segment_count_peak_delta": 440, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.523444Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..078c1a4f4b78e06841a13680781ed83f1964e3ce --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e81a30934d574b230fd9fcb7a0b0a469de4855c57b57d22ecb26f7ff3030996 +size 11659206 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0caf5f470aa8440fae63392a8938484b3295f162 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409335808, + "device": 2, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.966674Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409335808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409335808, + "device": 2, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.550555Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142943977472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142943977472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3336, + "segment_count_current_delta": 0, + "segment_count_peak": 3336, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.990538Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac2edd3c3f58413a28e8d894ed5e249b9ebfaa72 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d065d6e7a7a58cce740e1eb2653e0a7fb3332951ac483721044a3a3280b0af5c +size 1024383 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a07626e4ec21b47a3677222680fd8d9209425b11 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409335808, + "device": 2, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:22.392470Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409335808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409335808, + "device": 2, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.992237Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142943977472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142943977472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3336, + "segment_count_current_delta": 0, + "segment_count_peak": 3336, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.413897Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0952ba9b09305916fe1548d93c25057a7ea67458 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d40a12d5ca3f1a42a5aeb79a48e78cc0e6b77a16c8a8a15c856f8f778fc1552 +size 1024414 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c319b4a66b284663c7ed1ca0a0dfdaf49babc21 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84639218688, + "active_bytes_current_delta": 41082396160, + "active_bytes_peak": 91614024192, + "active_bytes_peak_delta": 48050910208, + "after": { + "active_bytes_current": 84639218688, + "active_bytes_peak": 91614024192, + "allocated_bytes_current": 84639218688, + "allocated_bytes_peak": 91614024192, + "device": 3, + "inactive_split_bytes_current": 4003202048, + "inactive_split_bytes_peak": 12193790976, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93308583936, + "reserved_bytes_peak": 93308583936, + "segment_count_current": 2209, + "segment_count_peak": 2209, + "timestamp_utc": "2026-08-26T18:57:14.715878Z", + "world_size": 8 + }, + "allocated_bytes_current": 84639218688, + "allocated_bytes_current_delta": 41082396160, + "allocated_bytes_peak": 91614024192, + "allocated_bytes_peak_delta": 48050910208, + "before": { + "active_bytes_current": 43556822528, + "active_bytes_peak": 43563113984, + "allocated_bytes_current": 43556822528, + "allocated_bytes_peak": 43563113984, + "device": 3, + "inactive_split_bytes_current": 1223664128, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T18:56:36.124364Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4003202048, + "inactive_split_bytes_current_delta": 2779537920, + "inactive_split_bytes_peak": 12193790976, + "inactive_split_bytes_peak_delta": 10580878336, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93308583936, + "reserved_bytes_current_delta": 48513417216, + "reserved_bytes_peak": 93308583936, + "reserved_bytes_peak_delta": 48513417216, + "segment_count_current": 2209, + "segment_count_current_delta": 1322, + "segment_count_peak": 2209, + "segment_count_peak_delta": 1322, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.911068Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..557bcf6ceef7fb3ff8686f3011fe0461827fa5c1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bbc327b800ea28c61c5b117e64d564f9029b195268fced181206659829598e5 +size 11336704 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb9669a129718874506286d353837db8d3519518 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84639218688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122380765184, + "active_bytes_peak_delta": 30766740992, + "after": { + "active_bytes_current": 84639218688, + "active_bytes_peak": 122380765184, + "allocated_bytes_current": 84639218688, + "allocated_bytes_peak": 122380765184, + "device": 3, + "inactive_split_bytes_current": 3904635904, + "inactive_split_bytes_peak": 12193790976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 124080095232, + "reserved_bytes_peak": 124080095232, + "segment_count_current": 2932, + "segment_count_peak": 2932, + "timestamp_utc": "2026-08-26T18:57:16.268107Z", + "world_size": 8 + }, + "allocated_bytes_current": 84639218688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122380765184, + "allocated_bytes_peak_delta": 30766740992, + "before": { + "active_bytes_current": 84639218688, + "active_bytes_peak": 91614024192, + "allocated_bytes_current": 84639218688, + "allocated_bytes_peak": 91614024192, + "device": 3, + "inactive_split_bytes_current": 4003202048, + "inactive_split_bytes_peak": 12193790976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93308583936, + "reserved_bytes_peak": 93308583936, + "segment_count_current": 2209, + "segment_count_peak": 2209, + "timestamp_utc": "2026-08-26T18:57:14.949036Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3904635904, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12193790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 124080095232, + "reserved_bytes_current_delta": 30771511296, + "reserved_bytes_peak": 124080095232, + "reserved_bytes_peak_delta": 30771511296, + "segment_count_current": 2932, + "segment_count_current_delta": 723, + "segment_count_peak": 2932, + "segment_count_peak_delta": 723, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.465288Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d9d23d6ca7e74fb42ddb04ce1a3b0c2b2de2617 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f24e01ae2a41576be2f51d98cdcca72d4e4107f29c84218a35eeeace42684f1e +size 10005096 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30b0338cd931ab686a166410bfb5963ee7638116 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84639218688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122385467392, + "active_bytes_peak_delta": 4702208, + "after": { + "active_bytes_current": 84639218688, + "active_bytes_peak": 122385467392, + "allocated_bytes_current": 84639218688, + "allocated_bytes_peak": 122385467392, + "device": 3, + "inactive_split_bytes_current": 4007396352, + "inactive_split_bytes_peak": 12193790976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 124080095232, + "reserved_bytes_peak": 124080095232, + "segment_count_current": 2932, + "segment_count_peak": 2932, + "timestamp_utc": "2026-08-26T18:57:17.636816Z", + "world_size": 8 + }, + "allocated_bytes_current": 84639218688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122385467392, + "allocated_bytes_peak_delta": 4702208, + "before": { + "active_bytes_current": 84639218688, + "active_bytes_peak": 122380765184, + "allocated_bytes_current": 84639218688, + "allocated_bytes_peak": 122380765184, + "device": 3, + "inactive_split_bytes_current": 3904635904, + "inactive_split_bytes_peak": 12193790976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 124080095232, + "reserved_bytes_peak": 124080095232, + "segment_count_current": 2932, + "segment_count_peak": 2932, + "timestamp_utc": "2026-08-26T18:57:16.493482Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4007396352, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 12193790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 124080095232, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 124080095232, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2932, + "segment_count_current_delta": 0, + "segment_count_peak": 2932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.835543Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13481ea0538f6d38fac3c44f6d6895ecdf69f623 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf995b39791b738cb9918361bf3dfeabbdb7755842c029e8c32e0679685a6dd1 +size 9983320 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc58aa5604fa314ca24e59eec5861440cdf254e2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84656262144, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129032920576, + "active_bytes_peak_delta": 6647453184, + "after": { + "active_bytes_current": 84656262144, + "active_bytes_peak": 129032920576, + "allocated_bytes_current": 84656262144, + "allocated_bytes_peak": 122385467392, + "device": 3, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_peak": 13104142336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142549712896, + "reserved_bytes_peak": 142549712896, + "segment_count_current": 3353, + "segment_count_peak": 3353, + "timestamp_utc": "2026-08-26T18:57:21.321480Z", + "world_size": 8 + }, + "allocated_bytes_current": 84656262144, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122385467392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84639218688, + "active_bytes_peak": 122385467392, + "allocated_bytes_current": 84639218688, + "allocated_bytes_peak": 122385467392, + "device": 3, + "inactive_split_bytes_current": 4007396352, + "inactive_split_bytes_peak": 12193790976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 124080095232, + "reserved_bytes_peak": 124080095232, + "segment_count_current": 2932, + "segment_count_peak": 2932, + "timestamp_utc": "2026-08-26T18:57:17.862945Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_current_delta": 35385344, + "inactive_split_bytes_peak": 13104142336, + "inactive_split_bytes_peak_delta": 910351360, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142549712896, + "reserved_bytes_current_delta": 18469617664, + "reserved_bytes_peak": 142549712896, + "reserved_bytes_peak_delta": 18469617664, + "segment_count_current": 3353, + "segment_count_current_delta": 421, + "segment_count_peak": 3353, + "segment_count_peak_delta": 421, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.533677Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d0e6263d271b0e1d8bd5d95f2d3552becf1e6bdc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40569b1347f21a5e50f7a875558837dc21417d48ee9975658ac7638eb7a0952f +size 13043697 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.json new file mode 100644 index 0000000000000000000000000000000000000000..306f8643b5ccd5ca77a2db1914b27f30cb9dbc80 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84656262144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129032920576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84656262144, + "active_bytes_peak": 129032920576, + "allocated_bytes_current": 84656262144, + "allocated_bytes_peak": 122385467392, + "device": 3, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_peak": 13104142336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142549712896, + "reserved_bytes_peak": 142549712896, + "segment_count_current": 3353, + "segment_count_peak": 3353, + "timestamp_utc": "2026-08-26T18:57:21.966676Z", + "world_size": 8 + }, + "allocated_bytes_current": 84656262144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122385467392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84656262144, + "active_bytes_peak": 129032920576, + "allocated_bytes_current": 84656262144, + "allocated_bytes_peak": 122385467392, + "device": 3, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_peak": 13104142336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142549712896, + "reserved_bytes_peak": 142549712896, + "segment_count_current": 3353, + "segment_count_peak": 3353, + "timestamp_utc": "2026-08-26T18:57:21.562190Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13104142336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142549712896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142549712896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3353, + "segment_count_current_delta": 0, + "segment_count_peak": 3353, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.991954Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0305962932a00e7a510b27542c096a7ea1cb3f49 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b05b8fd9cfb442a68ac108921b2a52dc9cb45f3ace93957b3d1a37262674f1d4 +size 1145785 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ad3ef449448513ccd7995530b488c0eda9badbc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84656262144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129032920576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84656262144, + "active_bytes_peak": 129032920576, + "allocated_bytes_current": 84656262144, + "allocated_bytes_peak": 122385467392, + "device": 3, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_peak": 13104142336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142549712896, + "reserved_bytes_peak": 142549712896, + "segment_count_current": 3353, + "segment_count_peak": 3353, + "timestamp_utc": "2026-08-26T18:57:22.392480Z", + "world_size": 8 + }, + "allocated_bytes_current": 84656262144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122385467392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84656262144, + "active_bytes_peak": 129032920576, + "allocated_bytes_current": 84656262144, + "allocated_bytes_peak": 122385467392, + "device": 3, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_peak": 13104142336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142549712896, + "reserved_bytes_peak": 142549712896, + "segment_count_current": 3353, + "segment_count_peak": 3353, + "timestamp_utc": "2026-08-26T18:57:21.993694Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4042781696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13104142336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142549712896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142549712896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3353, + "segment_count_current_delta": 0, + "segment_count_peak": 3353, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.415105Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8145c7858268ead925aed49b2701737f86baa6d3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f80a025f020d7c7fa17f8345eb5bb620908d07eea41fae6730f7901da0fd6d3 +size 1145785 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef940db20a8d99de7bfd8ffa8cc5c9be3a4accd2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620231168, + "active_bytes_current_delta": 41069553664, + "active_bytes_peak": 91608373760, + "active_bytes_peak_delta": 48051404800, + "after": { + "active_bytes_current": 84620231168, + "active_bytes_peak": 91608373760, + "allocated_bytes_current": 84620231168, + "allocated_bytes_peak": 91608373760, + "device": 4, + "inactive_split_bytes_current": 4173184512, + "inactive_split_bytes_peak": 12389535744, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93308583936, + "reserved_bytes_peak": 93308583936, + "segment_count_current": 2199, + "segment_count_peak": 2199, + "timestamp_utc": "2026-08-26T18:57:14.715883Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620231168, + "allocated_bytes_current_delta": 41069553664, + "allocated_bytes_peak": 91608373760, + "allocated_bytes_peak_delta": 48051404800, + "before": { + "active_bytes_current": 43550677504, + "active_bytes_peak": 43556968960, + "allocated_bytes_current": 43550677504, + "allocated_bytes_peak": 43556968960, + "device": 4, + "inactive_split_bytes_current": 1252877824, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T18:56:36.124297Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4173184512, + "inactive_split_bytes_current_delta": 2920306688, + "inactive_split_bytes_peak": 12389535744, + "inactive_split_bytes_peak_delta": 10776623104, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93308583936, + "reserved_bytes_current_delta": 48488251392, + "reserved_bytes_peak": 93308583936, + "reserved_bytes_peak_delta": 48488251392, + "segment_count_current": 2199, + "segment_count_current_delta": 1310, + "segment_count_peak": 2199, + "segment_count_peak_delta": 1310, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.907724Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e53e920b14f08ad03b44d5b5ce83889ec3eeb3c0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:808df123683f0e4d1437cfc18860e38f6b8f0e2a1b8f889185f461ff7d43a04a +size 10265593 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f0c37560b765b825c074069579afed18acdb280 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620231168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122397165568, + "active_bytes_peak_delta": 30788791808, + "after": { + "active_bytes_current": 84620231168, + "active_bytes_peak": 122397165568, + "allocated_bytes_current": 84620231168, + "allocated_bytes_peak": 122397165568, + "device": 4, + "inactive_split_bytes_current": 4074618368, + "inactive_split_bytes_peak": 12389535744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 124111552512, + "reserved_bytes_peak": 124111552512, + "segment_count_current": 2919, + "segment_count_peak": 2919, + "timestamp_utc": "2026-08-26T18:57:16.268101Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620231168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122397165568, + "allocated_bytes_peak_delta": 30788791808, + "before": { + "active_bytes_current": 84620231168, + "active_bytes_peak": 91608373760, + "allocated_bytes_current": 84620231168, + "allocated_bytes_peak": 91608373760, + "device": 4, + "inactive_split_bytes_current": 4173184512, + "inactive_split_bytes_peak": 12389535744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93308583936, + "reserved_bytes_peak": 93308583936, + "segment_count_current": 2199, + "segment_count_peak": 2199, + "timestamp_utc": "2026-08-26T18:57:14.946209Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4074618368, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12389535744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 124111552512, + "reserved_bytes_current_delta": 30802968576, + "reserved_bytes_peak": 124111552512, + "reserved_bytes_peak_delta": 30802968576, + "segment_count_current": 2919, + "segment_count_current_delta": 720, + "segment_count_peak": 2919, + "segment_count_peak_delta": 720, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.464169Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..285905b84ed4f50828b3d0340138570367e37202 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9546a3cdedc19d1379af2b642d67f7f7f9fe8f9d25155d6e6c41a6354f1d167e +size 9964097 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4dd4e00ba715b830619d8123e03b61593513e98a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620231168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122566591488, + "active_bytes_peak_delta": 169425920, + "after": { + "active_bytes_current": 84620231168, + "active_bytes_peak": 122566591488, + "allocated_bytes_current": 84620231168, + "allocated_bytes_peak": 122415596544, + "device": 4, + "inactive_split_bytes_current": 4177378816, + "inactive_split_bytes_peak": 12389535744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 124111552512, + "reserved_bytes_peak": 124111552512, + "segment_count_current": 2919, + "segment_count_peak": 2919, + "timestamp_utc": "2026-08-26T18:57:17.636821Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620231168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122415596544, + "allocated_bytes_peak_delta": 18430976, + "before": { + "active_bytes_current": 84620231168, + "active_bytes_peak": 122397165568, + "allocated_bytes_current": 84620231168, + "allocated_bytes_peak": 122397165568, + "device": 4, + "inactive_split_bytes_current": 4074618368, + "inactive_split_bytes_peak": 12389535744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 124111552512, + "reserved_bytes_peak": 124111552512, + "segment_count_current": 2919, + "segment_count_peak": 2919, + "timestamp_utc": "2026-08-26T18:57:16.492021Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4177378816, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 12389535744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 124111552512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 124111552512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2919, + "segment_count_current_delta": 0, + "segment_count_peak": 2919, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.832101Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3f4423313c96188322bc74ad775c02bb6a23a02 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a80831a0dd49d6d99c7c3777f30280b56c64c598e12b9b44375e59e393488bdf +size 9942396 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e09512ec99a4832a44c115f0df22a1da79fa275 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637274624, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129013933056, + "active_bytes_peak_delta": 6447341568, + "after": { + "active_bytes_current": 84637274624, + "active_bytes_peak": 129013933056, + "allocated_bytes_current": 84637274624, + "allocated_bytes_peak": 122415596544, + "device": 4, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_peak": 13276221952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142700707840, + "reserved_bytes_peak": 142700707840, + "segment_count_current": 3344, + "segment_count_peak": 3344, + "timestamp_utc": "2026-08-26T18:57:21.321529Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637274624, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122415596544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84620231168, + "active_bytes_peak": 122566591488, + "allocated_bytes_current": 84620231168, + "allocated_bytes_peak": 122415596544, + "device": 4, + "inactive_split_bytes_current": 4177378816, + "inactive_split_bytes_peak": 12389535744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 124111552512, + "reserved_bytes_peak": 124111552512, + "segment_count_current": 2919, + "segment_count_peak": 2919, + "timestamp_utc": "2026-08-26T18:57:17.859724Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_current_delta": 35385344, + "inactive_split_bytes_peak": 13276221952, + "inactive_split_bytes_peak_delta": 886686208, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142700707840, + "reserved_bytes_current_delta": 18589155328, + "reserved_bytes_peak": 142700707840, + "reserved_bytes_peak_delta": 18589155328, + "segment_count_current": 3344, + "segment_count_current_delta": 425, + "segment_count_peak": 3344, + "segment_count_peak_delta": 425, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.531953Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2680b3ab23480f6c99b7e0753cb53ed25e3fbe19 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbccb966f0bc7c697ff4f83958c98600e6bf894d49398b6d8004b29286261149 +size 13044392 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13efdf5239ab4403709e53448bd240b05974ecb3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637274624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129013933056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637274624, + "active_bytes_peak": 129013933056, + "allocated_bytes_current": 84637274624, + "allocated_bytes_peak": 122415596544, + "device": 4, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_peak": 13276221952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142700707840, + "reserved_bytes_peak": 142700707840, + "segment_count_current": 3344, + "segment_count_peak": 3344, + "timestamp_utc": "2026-08-26T18:57:21.966680Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637274624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122415596544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637274624, + "active_bytes_peak": 129013933056, + "allocated_bytes_current": 84637274624, + "allocated_bytes_peak": 122415596544, + "device": 4, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_peak": 13276221952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142700707840, + "reserved_bytes_peak": 142700707840, + "segment_count_current": 3344, + "segment_count_peak": 3344, + "timestamp_utc": "2026-08-26T18:57:21.560891Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13276221952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142700707840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142700707840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3344, + "segment_count_current_delta": 0, + "segment_count_peak": 3344, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.992261Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35668141af9b725e0f26f6f8be6c34232accd808 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32ca825932a591126c68aa69b765727b43e77abb7ea5e931820e0d7e6a33c27e +size 1146117 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dcaf77625d870e8c32e1a363776c6dfce82f4d83 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637274624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129013933056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637274624, + "active_bytes_peak": 129013933056, + "allocated_bytes_current": 84637274624, + "allocated_bytes_peak": 122415596544, + "device": 4, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_peak": 13276221952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142700707840, + "reserved_bytes_peak": 142700707840, + "segment_count_current": 3344, + "segment_count_peak": 3344, + "timestamp_utc": "2026-08-26T18:57:22.392443Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637274624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122415596544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637274624, + "active_bytes_peak": 129013933056, + "allocated_bytes_current": 84637274624, + "allocated_bytes_peak": 122415596544, + "device": 4, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_peak": 13276221952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142700707840, + "reserved_bytes_peak": 142700707840, + "segment_count_current": 3344, + "segment_count_peak": 3344, + "timestamp_utc": "2026-08-26T18:57:21.994010Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4212764160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13276221952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142700707840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142700707840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3344, + "segment_count_current_delta": 0, + "segment_count_peak": 3344, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.415679Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b7446a830b66e3bfd59bf5d50f64da2183556bb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4a812c8c9953fd12d61a9fa1a3c060b55cf99bd397c61e08fddd1e5740f5aa +size 1146117 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.json new file mode 100644 index 0000000000000000000000000000000000000000..efa73729198b664f22875f3959241a09190d0fdc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 41069773824, + "active_bytes_peak": 91617038336, + "active_bytes_peak_delta": 48057203712, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 91617038336, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 91617038336, + "device": 5, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_peak": 12182193664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93296001024, + "reserved_bytes_peak": 93296001024, + "segment_count_current": 2187, + "segment_count_peak": 2187, + "timestamp_utc": "2026-08-26T18:57:14.715850Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 41069773824, + "allocated_bytes_peak": 91617038336, + "allocated_bytes_peak_delta": 48057203712, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 5, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:56:36.124260Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_current_delta": 3056401408, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 10569281024, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93296001024, + "reserved_bytes_current_delta": 48460988416, + "reserved_bytes_peak": 93296001024, + "reserved_bytes_peak_delta": 48460988416, + "segment_count_current": 2187, + "segment_count_current_delta": 1304, + "segment_count_peak": 2187, + "segment_count_peak_delta": 1304, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.907500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e95e2ad24b208c8f18e5f8c4f83313f1eed2e84 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed1b5fc9e78f2e9ca4e6fc8395b8a81c19d4cf802bf1f36ef818da1c4d22d8d +size 9776530 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4dda1594468061fbce8d1a962d33137ff9f46373 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122552418304, + "active_bytes_peak_delta": 30935379968, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122552418304, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122401423360, + "device": 5, + "inactive_split_bytes_current": 4314802176, + "inactive_split_bytes_peak": 12182193664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:16.268085Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122401423360, + "allocated_bytes_peak_delta": 30784385024, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 91617038336, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 91617038336, + "device": 5, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_peak": 12182193664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93296001024, + "reserved_bytes_peak": 93296001024, + "segment_count_current": 2187, + "segment_count_peak": 2187, + "timestamp_utc": "2026-08-26T18:57:14.941036Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4314802176, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 124092678144, + "reserved_bytes_current_delta": 30796677120, + "reserved_bytes_peak": 124092678144, + "reserved_bytes_peak_delta": 30796677120, + "segment_count_current": 2896, + "segment_count_current_delta": 709, + "segment_count_peak": 2896, + "segment_count_peak_delta": 709, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.452271Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d2f2869366225d7343eb72c22e76f82b7b156d2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbbeb24eacfb24bdbd734c5ff0663e0aacb114568394a159437e97c2de5818cb +size 9503153 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9545a239063d3b25a6a90ef129b76e6504ebac2b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122862844928, + "active_bytes_peak_delta": 310426624, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122862844928, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122409860096, + "device": 5, + "inactive_split_bytes_current": 4518225920, + "inactive_split_bytes_peak": 12182193664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:17.636782Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 8436736, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122552418304, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122401423360, + "device": 5, + "inactive_split_bytes_current": 4314802176, + "inactive_split_bytes_peak": 12182193664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:16.476772Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4518225920, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 124092678144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 124092678144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2896, + "segment_count_current_delta": 0, + "segment_count_peak": 2896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.824669Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16986ec595f9194a5645e72278db9a3a7331ce8f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e53e1b1d1fd8cded3d575fbc8221f1e543225d075d6c45b4726535d8c3f70dad +size 9478730 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18dd0cec50cda7dc6a8de621ea9df5b4acbc7e39 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 6154173952, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 5, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 143044640768, + "reserved_bytes_peak": 143044640768, + "segment_count_current": 3337, + "segment_count_peak": 3337, + "timestamp_utc": "2026-08-26T18:57:21.321493Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122862844928, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122409860096, + "device": 5, + "inactive_split_bytes_current": 4518225920, + "inactive_split_bytes_peak": 12182193664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:17.848873Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": -168038400, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 1132885504, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 143044640768, + "reserved_bytes_current_delta": 18951962624, + "reserved_bytes_peak": 143044640768, + "reserved_bytes_peak_delta": 18951962624, + "segment_count_current": 3337, + "segment_count_current_delta": 441, + "segment_count_peak": 3337, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.526139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3dae04ce819f36d08061eafeaea16917bf4f8da8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5504ac8adb670b9172093368997137b8ed9323e12821d78bc9ee7443a98aa8f +size 11658449 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185721967078Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185721967078Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2076e4cd379216071e534c6dc51821fd89968825 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185721967078Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 5, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 143044640768, + "reserved_bytes_peak": 143044640768, + "segment_count_current": 3337, + "segment_count_peak": 3337, + "timestamp_utc": "2026-08-26T18:57:21.966657Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 5, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 143044640768, + "reserved_bytes_peak": 143044640768, + "segment_count_current": 3337, + "segment_count_peak": 3337, + "timestamp_utc": "2026-08-26T18:57:21.552578Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 143044640768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143044640768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3337, + "segment_count_current_delta": 0, + "segment_count_peak": 3337, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185721967078Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.998276Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185721967078Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185721967078Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d510ad03ff48776661082501d0e9525b190e4f5a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185721967078Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcfcb7cc811aabbe4e41f27edf230abbe1724e1144b14e5556a60e0608e5ebd7 +size 1024089 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185722392834Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185722392834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65f6712a2f3c6320db69e05a6c1de227de5fe68a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185722392834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 5, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 143044640768, + "reserved_bytes_peak": 143044640768, + "segment_count_current": 3337, + "segment_count_peak": 3337, + "timestamp_utc": "2026-08-26T18:57:22.392445Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 5, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 143044640768, + "reserved_bytes_peak": 143044640768, + "segment_count_current": 3337, + "segment_count_peak": 3337, + "timestamp_utc": "2026-08-26T18:57:22.000016Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 143044640768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143044640768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3337, + "segment_count_current_delta": 0, + "segment_count_peak": 3337, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185722392834Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.414787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185722392834Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185722392834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da7cede3809e153fcdccc11ca833beeae630b82a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185722392834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eac377dfea4f439256d945d6a1994f155b2091063bed1a6924583985d44dbf2 +size 1024120 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185714716423Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185714716423Z.json new file mode 100644 index 0000000000000000000000000000000000000000..193ba4d76e3a00f11329f7093459e7a9e9c679ee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185714716423Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84631096320, + "active_bytes_current_delta": 41077685760, + "active_bytes_peak": 91615204864, + "active_bytes_peak_delta": 48055502848, + "after": { + "active_bytes_current": 84631096320, + "active_bytes_peak": 91615204864, + "allocated_bytes_current": 84631096320, + "allocated_bytes_peak": 91615204864, + "device": 6, + "inactive_split_bytes_current": 4149736448, + "inactive_split_bytes_peak": 12049629184, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93296001024, + "reserved_bytes_peak": 93296001024, + "segment_count_current": 2210, + "segment_count_peak": 2210, + "timestamp_utc": "2026-08-26T18:57:14.715966Z", + "world_size": 8 + }, + "allocated_bytes_current": 84631096320, + "allocated_bytes_current_delta": 41077685760, + "allocated_bytes_peak": 91615204864, + "allocated_bytes_peak_delta": 48055502848, + "before": { + "active_bytes_current": 43553410560, + "active_bytes_peak": 43559702016, + "allocated_bytes_current": 43553410560, + "allocated_bytes_peak": 43559702016, + "device": 6, + "inactive_split_bytes_current": 1256436224, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:56:36.124458Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4149736448, + "inactive_split_bytes_current_delta": 2893300224, + "inactive_split_bytes_peak": 12049629184, + "inactive_split_bytes_peak_delta": 10436716544, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93296001024, + "reserved_bytes_current_delta": 48471474176, + "reserved_bytes_peak": 93296001024, + "reserved_bytes_peak_delta": 48471474176, + "segment_count_current": 2210, + "segment_count_current_delta": 1320, + "segment_count_peak": 2210, + "segment_count_peak_delta": 1320, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185714716423Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.906915Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185714716423Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185714716423Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..739ca95b27ce437bd30895f6445718e1c82353ad --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185714716423Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:151f5e21fd78e2a646362e32015f6db93ace0e0cebd72019521ebff3dcd32fe1 +size 11389018 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185716268542Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185716268542Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ce45dd247f046ea3f1d6061c5b95971aec9f9fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185716268542Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84631096320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122393898496, + "active_bytes_peak_delta": 30778693632, + "after": { + "active_bytes_current": 84631096320, + "active_bytes_peak": 122393898496, + "allocated_bytes_current": 84631096320, + "allocated_bytes_peak": 122393898496, + "device": 6, + "inactive_split_bytes_current": 4051170304, + "inactive_split_bytes_peak": 12049629184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 124067512320, + "reserved_bytes_peak": 124067512320, + "segment_count_current": 2933, + "segment_count_peak": 2933, + "timestamp_utc": "2026-08-26T18:57:16.268110Z", + "world_size": 8 + }, + "allocated_bytes_current": 84631096320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122393898496, + "allocated_bytes_peak_delta": 30778693632, + "before": { + "active_bytes_current": 84631096320, + "active_bytes_peak": 91615204864, + "allocated_bytes_current": 84631096320, + "allocated_bytes_peak": 91615204864, + "device": 6, + "inactive_split_bytes_current": 4149736448, + "inactive_split_bytes_peak": 12049629184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93296001024, + "reserved_bytes_peak": 93296001024, + "segment_count_current": 2210, + "segment_count_peak": 2210, + "timestamp_utc": "2026-08-26T18:57:14.941422Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4051170304, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12049629184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 124067512320, + "reserved_bytes_current_delta": 30771511296, + "reserved_bytes_peak": 124067512320, + "reserved_bytes_peak_delta": 30771511296, + "segment_count_current": 2933, + "segment_count_current_delta": 723, + "segment_count_peak": 2933, + "segment_count_peak_delta": 723, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185716268542Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.456262Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185716268542Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185716268542Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..941a46bce10e062366b3c74dcc57302901dd7727 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185716268542Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:944531d604ff2fb5b8dc3e7cdd03086d1f2a2b23fae730e8b88a9774eb3be06e +size 10006258 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185717637258Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185717637258Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a67d5faf2590c0e6b5ce35583e40d3b1c4c86acf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185717637258Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84631096320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122702228992, + "active_bytes_peak_delta": 308330496, + "after": { + "active_bytes_current": 84631096320, + "active_bytes_peak": 122702228992, + "allocated_bytes_current": 84631096320, + "allocated_bytes_peak": 122400239104, + "device": 6, + "inactive_split_bytes_current": 4254594048, + "inactive_split_bytes_peak": 12049629184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 124067512320, + "reserved_bytes_peak": 124067512320, + "segment_count_current": 2933, + "segment_count_peak": 2933, + "timestamp_utc": "2026-08-26T18:57:17.636832Z", + "world_size": 8 + }, + "allocated_bytes_current": 84631096320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122400239104, + "allocated_bytes_peak_delta": 6340608, + "before": { + "active_bytes_current": 84631096320, + "active_bytes_peak": 122393898496, + "allocated_bytes_current": 84631096320, + "allocated_bytes_peak": 122393898496, + "device": 6, + "inactive_split_bytes_current": 4051170304, + "inactive_split_bytes_peak": 12049629184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 124067512320, + "reserved_bytes_peak": 124067512320, + "segment_count_current": 2933, + "segment_count_peak": 2933, + "timestamp_utc": "2026-08-26T18:57:16.480770Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4254594048, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 12049629184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 124067512320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 124067512320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2933, + "segment_count_current_delta": 0, + "segment_count_peak": 2933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185717637258Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.824795Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185717637258Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185717637258Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8230ba6b46b06f3cd289acd1cde7d50d6eb5a3dd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185717637258Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db9cdbb82c2f68ad00ef2f4caaf7e192d1970be6850f6008a54804af16d16d96 +size 9984557 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185721321969Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185721321969Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63288b227a02dd7e91bea940988ffaf04ff12265 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185721321969Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84648139776, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129024798208, + "active_bytes_peak_delta": 6322569216, + "after": { + "active_bytes_current": 84648139776, + "active_bytes_peak": 129024798208, + "allocated_bytes_current": 84648139776, + "allocated_bytes_peak": 122400239104, + "device": 6, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_peak": 13152110592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142788788224, + "reserved_bytes_peak": 142788788224, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.321465Z", + "world_size": 8 + }, + "allocated_bytes_current": 84648139776, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122400239104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84631096320, + "active_bytes_peak": 122702228992, + "allocated_bytes_current": 84631096320, + "allocated_bytes_peak": 122400239104, + "device": 6, + "inactive_split_bytes_current": 4254594048, + "inactive_split_bytes_peak": 12049629184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 124067512320, + "reserved_bytes_peak": 124067512320, + "segment_count_current": 2933, + "segment_count_peak": 2933, + "timestamp_utc": "2026-08-26T18:57:17.849694Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_current_delta": -65277952, + "inactive_split_bytes_peak": 13152110592, + "inactive_split_bytes_peak_delta": 1102481408, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142788788224, + "reserved_bytes_current_delta": 18721275904, + "reserved_bytes_peak": 142788788224, + "reserved_bytes_peak_delta": 18721275904, + "segment_count_current": 3356, + "segment_count_current_delta": 423, + "segment_count_peak": 3356, + "segment_count_peak_delta": 423, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185721321969Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.525673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185721321969Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185721321969Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c5aae2d5b0f9b94195c0ddbc6154381451218ca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185721321969Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:995af63e3e82591a628c33068ac8670941b0e2eb552038e25cbd4afd015048fa +size 13045138 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185721967115Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185721967115Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56b40e1afdc6062a5c9b120debb595e05df5480a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185721967115Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84648139776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129024798208, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84648139776, + "active_bytes_peak": 129024798208, + "allocated_bytes_current": 84648139776, + "allocated_bytes_peak": 122400239104, + "device": 6, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_peak": 13152110592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142788788224, + "reserved_bytes_peak": 142788788224, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.966683Z", + "world_size": 8 + }, + "allocated_bytes_current": 84648139776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122400239104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84648139776, + "active_bytes_peak": 129024798208, + "allocated_bytes_current": 84648139776, + "allocated_bytes_peak": 122400239104, + "device": 6, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_peak": 13152110592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142788788224, + "reserved_bytes_peak": 142788788224, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.552579Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13152110592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142788788224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142788788224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3356, + "segment_count_current_delta": 0, + "segment_count_peak": 3356, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185721967115Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.991182Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185721967115Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185721967115Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9191eed70001b0f880b4bc3d8c0b8a27b5e79486 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185721967115Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5649f6613f3a6dc5ba46a834bd32dc94a30e5dc6d901383348d4517cdf30439d +size 1146984 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185722392841Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185722392841Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef2d6bd0cdd6f65eb4c17d7569f221383d6eed12 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185722392841Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84648139776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129024798208, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84648139776, + "active_bytes_peak": 129024798208, + "allocated_bytes_current": 84648139776, + "allocated_bytes_peak": 122400239104, + "device": 6, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_peak": 13152110592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142788788224, + "reserved_bytes_peak": 142788788224, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:22.392452Z", + "world_size": 8 + }, + "allocated_bytes_current": 84648139776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122400239104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84648139776, + "active_bytes_peak": 129024798208, + "allocated_bytes_current": 84648139776, + "allocated_bytes_peak": 122400239104, + "device": 6, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_peak": 13152110592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142788788224, + "reserved_bytes_peak": 142788788224, + "segment_count_current": 3356, + "segment_count_peak": 3356, + "timestamp_utc": "2026-08-26T18:57:21.992858Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4189316096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13152110592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142788788224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142788788224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3356, + "segment_count_current_delta": 0, + "segment_count_peak": 3356, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185722392841Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.414555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185722392841Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185722392841Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3650926507315650c544196c9382783bdf4d2605 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185722392841Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11e65c2010c3510471ab39b50a1f7ed731d7401ce7a2626ebcaf6b803fc02f9f +size 1146984 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185714716526Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185714716526Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b1dc87d6ddc27b68f74090af6feefd78fbd79be --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185714716526Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 41069773824, + "active_bytes_peak": 91617038336, + "active_bytes_peak_delta": 48057203712, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 91617038336, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 91617038336, + "device": 7, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_peak": 12182193664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 92490694656, + "reserved_bytes_peak": 92490694656, + "segment_count_current": 2179, + "segment_count_peak": 2179, + "timestamp_utc": "2026-08-26T18:57:14.716045Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 41069773824, + "allocated_bytes_peak": 91617038336, + "allocated_bytes_peak_delta": 48057203712, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 7, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:56:36.124289Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_current_delta": 3056401408, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 10569281024, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 92490694656, + "reserved_bytes_current_delta": 47655682048, + "reserved_bytes_peak": 92490694656, + "reserved_bytes_peak_delta": 47655682048, + "segment_count_current": 2179, + "segment_count_current_delta": 1296, + "segment_count_peak": 2179, + "segment_count_peak_delta": 1296, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185714716526Z.pickle", + "timestamp_utc": "2026-08-26T18:57:14.897110Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185714716526Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185714716526Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e0cce31a59eba2cf54f15dfc389c0adaad66095 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185714716526Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d83f41906c2db35dae7000a1b0eb157bb5a1fe532b7f66b8620f3971f0a56f9 +size 9773990 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185716268786Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185716268786Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3be7914066d80daf506c827dbc236d6309699a65 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185716268786Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122552299520, + "active_bytes_peak_delta": 30935261184, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122552299520, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122401423360, + "device": 7, + "inactive_split_bytes_current": 4314802176, + "inactive_split_bytes_peak": 12182193664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 123589361664, + "reserved_bytes_peak": 123589361664, + "segment_count_current": 2891, + "segment_count_peak": 2891, + "timestamp_utc": "2026-08-26T18:57:16.268344Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122401423360, + "allocated_bytes_peak_delta": 30784385024, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 91617038336, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 91617038336, + "device": 7, + "inactive_split_bytes_current": 4312705024, + "inactive_split_bytes_peak": 12182193664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 92490694656, + "reserved_bytes_peak": 92490694656, + "segment_count_current": 2179, + "segment_count_peak": 2179, + "timestamp_utc": "2026-08-26T18:57:14.925658Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4314802176, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 123589361664, + "reserved_bytes_current_delta": 31098667008, + "reserved_bytes_peak": 123589361664, + "reserved_bytes_peak_delta": 31098667008, + "segment_count_current": 2891, + "segment_count_current_delta": 712, + "segment_count_peak": 2891, + "segment_count_peak_delta": 712, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185716268786Z.pickle", + "timestamp_utc": "2026-08-26T18:57:16.444947Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185716268786Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185716268786Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b708a4a00ac089f81027f29460e9fc2700a9d50 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185716268786Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e863e507e2f3bcbdfd706ea7b6cf7d7a5260f7b657cdb048f8f56db878e9ff +size 9502768 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185717637292Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185717637292Z.json new file mode 100644 index 0000000000000000000000000000000000000000..703cf78aaef7c9d840380e227834e837b742c19d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185717637292Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84623316992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122862844928, + "active_bytes_peak_delta": 310545408, + "after": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122862844928, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122409860096, + "device": 7, + "inactive_split_bytes_current": 4417562624, + "inactive_split_bytes_peak": 12182193664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:17.636870Z", + "world_size": 8 + }, + "allocated_bytes_current": 84623316992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 8436736, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122552299520, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122401423360, + "device": 7, + "inactive_split_bytes_current": 4314802176, + "inactive_split_bytes_peak": 12182193664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 123589361664, + "reserved_bytes_peak": 123589361664, + "segment_count_current": 2891, + "segment_count_peak": 2891, + "timestamp_utc": "2026-08-26T18:57:16.465279Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4417562624, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 12182193664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 124092678144, + "reserved_bytes_current_delta": 503316480, + "reserved_bytes_peak": 124092678144, + "reserved_bytes_peak_delta": 503316480, + "segment_count_current": 2896, + "segment_count_current_delta": 5, + "segment_count_peak": 2896, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185717637292Z.pickle", + "timestamp_utc": "2026-08-26T18:57:17.817685Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185717637292Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185717637292Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a8f469b132f3fbb888c501723ff5e3169781d5e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185717637292Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff704a6a3e0f201bf12256fe0c1fd6970e8424aa6c1d796f40b9f1c95e304a03 +size 9658084 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185721321975Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185721321975Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b782a5411e31eadacc5439b4661421d9ecb9450 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185721321975Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 6154173952, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 7, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.321465Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84623316992, + "active_bytes_peak": 122862844928, + "allocated_bytes_current": 84623316992, + "allocated_bytes_peak": 122409860096, + "device": 7, + "inactive_split_bytes_current": 4417562624, + "inactive_split_bytes_peak": 12182193664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 124092678144, + "reserved_bytes_peak": 124092678144, + "segment_count_current": 2896, + "segment_count_peak": 2896, + "timestamp_utc": "2026-08-26T18:57:17.837133Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": -67375104, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 1132885504, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142943977472, + "reserved_bytes_current_delta": 18851299328, + "reserved_bytes_peak": 142943977472, + "reserved_bytes_peak_delta": 18851299328, + "segment_count_current": 3336, + "segment_count_current_delta": 440, + "segment_count_peak": 3336, + "segment_count_peak_delta": 440, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185721321975Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.520588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185721321975Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185721321975Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f2a5c09d0ad33944adee66c9b68fa79399f251e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185721321975Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63762fcdf0a412a3315597564904d62c470e6b9569c2266af8887d2051089c5c +size 11659529 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185721967099Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185721967099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..507f899ec44d3efbe4fbc79d4cc42d38871bdf92 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185721967099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 7, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.966674Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 7, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.542378Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142943977472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142943977472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3336, + "segment_count_current_delta": 0, + "segment_count_peak": 3336, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185721967099Z.pickle", + "timestamp_utc": "2026-08-26T18:57:21.988833Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185721967099Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185721967099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a4bbf14e3aafd5ee15940c8a874e2c050682d83b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185721967099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:757784a7c4b12b1d995ec0b6dc38280a63eae7d6d008878c6e437462e63692e5 +size 1024997 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185722392880Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185722392880Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d61e4d1ccc150c2d13ab1744c67c75d6c60729fd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185722392880Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84640360448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 129017018880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 7, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:22.392494Z", + "world_size": 8 + }, + "allocated_bytes_current": 84640360448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 122409860096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84640360448, + "active_bytes_peak": 129017018880, + "allocated_bytes_current": 84640360448, + "allocated_bytes_peak": 122409860096, + "device": 7, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_peak": 13315079168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142943977472, + "reserved_bytes_peak": 142943977472, + "segment_count_current": 3336, + "segment_count_peak": 3336, + "timestamp_utc": "2026-08-26T18:57:21.990522Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4350187520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13315079168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142943977472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142943977472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3336, + "segment_count_current_delta": 0, + "segment_count_peak": 3336, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185722392880Z.pickle", + "timestamp_utc": "2026-08-26T18:57:22.414379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185722392880Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185722392880Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c4bc1e735cdbd581c44bde2a4636d774d79f6dc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185722392880Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3539455d76da727e6e86e76e5c096e4311258691a6064006b8919ab15e9fae07 +size 1025028 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..4007d5c2b0742d1c55a92fe85bd4beaea62b8c38 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.125 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..13a083d5cecdfb3ed7be188b06f12a1927555cae --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log @@ -0,0 +1,2835 @@ +[18:56:08] WARN: better-memory-dump-support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.125 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 18:56:10.295000 582093 torch/distributed/run.py:766] +W0826 18:56:10.295000 582093 torch/distributed/run.py:766] ***************************************** +W0826 18:56:10.295000 582093 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:56:10.295000 582093 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1 is connected to 73 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 27456 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777 + + + + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 3 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00[Gloo] Rank is connected to peer ranks. 00Expected number of connected peer ranks is : is connected to peer ranks. 00Expected number of connected peer ranks is : + peer ranks. [Gloo] Rank 0Expected number of connected peer ranks is : +00 is connected to +0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 18:56:28.483538872 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:56:28.483548589 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:56:28.483623481 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:56:28.483658435 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:56:28.487421416 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:56:28.487448487 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:56:28.487560549 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 18:56:28.633373987 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.094 seconds +time to initialize megatron (seconds): 7.273 +[after megatron is initialized] datetime: 2026-08-26 18:56:34 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.mlp.router.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.k_layernorm.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.linear_proj.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_proj.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.k_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.q_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:56:35 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 18:56:36 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (1191.90, 1318.27) + train/valid/test-data-iterators-setup ..........: (63.99, 68.68) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 18:56:36 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 18:57:14] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 38591.2 | throughput per GPU (TFLOP/s/GPU): 2.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095255E+01 | loss scale: 1.0 | grad norm: 54.344 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 80700.50390625 | max allocated: 87369.37841796875 | reserved: 88968.0 | max reserved: 88968.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 18:57:16] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 1552.6 | throughput per GPU (TFLOP/s/GPU): 57.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095489E+01 | loss scale: 1.0 | grad norm: 56.495 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:57:17] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 1368.6 | throughput per GPU (TFLOP/s/GPU): 65.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008457E+01 | loss scale: 1.0 | grad norm: 179.141 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 18:57:21] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 3684.6 | throughput per GPU (TFLOP/s/GPU): 24.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.603754E+00 | loss scale: 1.0 | grad norm: 8.774 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:57:21] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 645.4 | throughput per GPU (TFLOP/s/GPU): 138.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.309366E+00 | loss scale: 1.0 | grad norm: 9.559 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:57:22] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 425.5 | throughput per GPU (TFLOP/s/GPU): 209.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.118230E+00 | loss scale: 1.0 | grad norm: 2.934 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:57:22 +[rank0]:[W826 18:57:23.782679635 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:57:28.981000 582093 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:57:28.982000 582093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 582186 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 582093 got signal: 15 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..5dca83b42f38077b618e718176928b1755ceb492 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 4 +3, 4 +4, 27 +5, 27 +6, 4 +7, 27 +0, 917 +1, 907 +2, 923 +3, 909 +4, 911 +5, 921 +6, 911 +7, 917 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 39225 +1, 38995 +2, 39541 +3, 42895 +4, 38995 +5, 39197 +6, 39007 +7, 39197 +0, 45171 +1, 45173 +2, 45183 +3, 45145 +4, 45169 +5, 45151 +6, 45173 +7, 45183 +0, 45897 +1, 45899 +2, 45909 +3, 45871 +4, 45895 +5, 45909 +6, 45899 +7, 45909 +0, 46919 +1, 46921 +2, 46931 +3, 46893 +4, 46917 +5, 46931 +6, 46921 +7, 46931 +0, 47015 +1, 47017 +2, 47027 +3, 46989 +4, 47013 +5, 47027 +6, 47017 +7, 47027 +0, 47177 +1, 47199 +2, 47197 +3, 47175 +4, 47177 +5, 47187 +6, 47195 +7, 47181 +0, 47879 +1, 47881 +2, 47891 +3, 47853 +4, 47877 +5, 47891 +6, 47881 +7, 47891 +0, 48039 +1, 48041 +2, 48051 +3, 48013 +4, 48037 +5, 48051 +6, 48041 +7, 48051 +0, 48515 +1, 48517 +2, 48531 +3, 48489 +4, 48513 +5, 48531 +6, 48477 +7, 48531 +0, 48739 +1, 48741 +2, 48751 +3, 48713 +4, 48737 +5, 48751 +6, 48701 +7, 48751 +0, 93903 +1, 93875 +2, 93901 +3, 93925 +4, 93911 +5, 93895 +6, 93871 +7, 93911 +0, 94411 +1, 94393 +2, 94397 +3, 94405 +4, 94409 +5, 94397 +6, 94393 +7, 94397 +0, 94411 +1, 94393 +2, 94397 +3, 94405 +4, 94409 +5, 94397 +6, 94393 +7, 94397 +0, 94411 +1, 94393 +2, 94397 +3, 94405 +4, 94409 +5, 94397 +6, 94393 +7, 94397 +0, 94411 +1, 94393 +2, 94397 +3, 94405 +4, 94409 +5, 94397 +6, 94393 +7, 94397 +0, 95701 +1, 95587 +2, 95399 +3, 95311 +4, 95219 +5, 95303 +6, 95395 +7, 95207 +0, 96349 +1, 96235 +2, 96047 +3, 95959 +4, 95867 +5, 95951 +6, 96043 +7, 95855 +0, 97033 +1, 96927 +2, 96747 +3, 96669 +4, 96583 +5, 96669 +6, 96709 +7, 96505 +0, 126409 +1, 126313 +2, 126319 +3, 126307 +4, 126157 +5, 126127 +6, 126083 +7, 126223 +0, 96045 +1, 96551 +2, 97873 +3, 101143 +4, 97321 +5, 97009 +6, 96611 +7, 101407 +0, 146943 +1, 147327 +2, 147661 +3, 147029 +4, 147567 +5, 147661 +6, 146857 +7, 147661 +0, 146943 +1, 147327 +2, 147661 +3, 147029 +4, 147567 +5, 147661 +6, 146857 +7, 147661 +0, 146127 +1, 146511 +2, 146845 +3, 2442 +4, 146751 +5, 146845 +6, 146041 +7, 146845 +0, 903 +1, 1162 +2, 146845 +3, 2186 +4, 906 +5, 906 +6, 906 +7, 906 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185834760934Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185834760934Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cde5b63530f5eba65a7166d3204fc72d27acaead --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185834760934Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620597248, + "active_bytes_current_delta": 41068995072, + "active_bytes_peak": 92805288960, + "active_bytes_peak_delta": 49247395328, + "after": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 92805288960, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 92805288960, + "device": 0, + "inactive_split_bytes_current": 6798452736, + "inactive_split_bytes_peak": 12487894016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 94340382720, + "reserved_bytes_peak": 94340382720, + "segment_count_current": 2268, + "segment_count_peak": 2268, + "timestamp_utc": "2026-08-26T18:58:34.760471Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620597248, + "allocated_bytes_current_delta": 41068995072, + "allocated_bytes_peak": 92805288960, + "allocated_bytes_peak_delta": 49247395328, + "before": { + "active_bytes_current": 43551602176, + "active_bytes_peak": 43557893632, + "allocated_bytes_current": 43551602176, + "allocated_bytes_peak": 43557893632, + "device": 0, + "inactive_split_bytes_current": 1256147456, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T18:57:56.845738Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6798452736, + "inactive_split_bytes_current_delta": 5542305280, + "inactive_split_bytes_peak": 12487894016, + "inactive_split_bytes_peak_delta": 10874981376, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 94340382720, + "reserved_bytes_current_delta": 49517953024, + "reserved_bytes_peak": 94340382720, + "reserved_bytes_peak_delta": 49517953024, + "segment_count_current": 2268, + "segment_count_current_delta": 1383, + "segment_count_peak": 2268, + "segment_count_peak_delta": 1383, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T185834760934Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.958346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185834760934Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185834760934Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..97b93fa539c63dfaddc03e77c68610ef02f37382 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T185834760934Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dd83b07b42d5cbc49194b31bf533e674662149c2d9ea45b57c5889b5084089a +size 9804973 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185836224576Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185836224576Z.json new file mode 100644 index 0000000000000000000000000000000000000000..714ad865b0fa53f0e04a80c9744f9e13dd0f8bfc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185836224576Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620597248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123525784576, + "active_bytes_peak_delta": 30720495616, + "after": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 123525784576, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6701983744, + "inactive_split_bytes_peak": 12487894016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 125044785152, + "reserved_bytes_peak": 125044785152, + "segment_count_current": 2906, + "segment_count_peak": 2906, + "timestamp_utc": "2026-08-26T18:58:36.224117Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620597248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123525784576, + "allocated_bytes_peak_delta": 30720495616, + "before": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 92805288960, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 92805288960, + "device": 0, + "inactive_split_bytes_current": 6798452736, + "inactive_split_bytes_peak": 12487894016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 94340382720, + "reserved_bytes_peak": 94340382720, + "segment_count_current": 2268, + "segment_count_peak": 2268, + "timestamp_utc": "2026-08-26T18:58:34.997495Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6701983744, + "inactive_split_bytes_current_delta": -96468992, + "inactive_split_bytes_peak": 12487894016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 125044785152, + "reserved_bytes_current_delta": 30704402432, + "reserved_bytes_peak": 125044785152, + "reserved_bytes_peak_delta": 30704402432, + "segment_count_current": 2906, + "segment_count_current_delta": 638, + "segment_count_peak": 2906, + "segment_count_peak_delta": 638, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T185836224576Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.439267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185836224576Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185836224576Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6b0d426aaefe2dd02c57bfd3b671a1b00704a58 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T185836224576Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd4cc966d9db2fb49f7726709fc3c7490ab884069022e331d61a7a39da19a66d +size 10139024 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185837639728Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185837639728Z.json new file mode 100644 index 0000000000000000000000000000000000000000..087aee9af62105a7d64dd647ef0138baa6e43e24 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185837639728Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84620597248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123525784576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 123525784576, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6802647040, + "inactive_split_bytes_peak": 12487894016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 125044785152, + "reserved_bytes_peak": 125044785152, + "segment_count_current": 2906, + "segment_count_peak": 2906, + "timestamp_utc": "2026-08-26T18:58:37.639296Z", + "world_size": 8 + }, + "allocated_bytes_current": 84620597248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123525784576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 123525784576, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6701983744, + "inactive_split_bytes_peak": 12487894016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 125044785152, + "reserved_bytes_peak": 125044785152, + "segment_count_current": 2906, + "segment_count_peak": 2906, + "timestamp_utc": "2026-08-26T18:58:36.477609Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6802647040, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 12487894016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 125044785152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125044785152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2906, + "segment_count_current_delta": 0, + "segment_count_peak": 2906, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T185837639728Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.835777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185837639728Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185837639728Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07e1d5b1502318c333709f3e60869368b5f22fbe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T185837639728Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a75b7f1bfdb77b1ad4019d7dd06cfc1604470089dff956a0bf08f0eb8ecaba24 +size 10123115 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185841332667Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185841332667Z.json new file mode 100644 index 0000000000000000000000000000000000000000..841d57f8b33c630d0f28df046ad341316ea62ea2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185841332667Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637640704, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130120022528, + "active_bytes_peak_delta": 6594237952, + "after": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 130120022528, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_peak": 16642824192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_peak": 146467192832, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:41.332156Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637640704, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123525784576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84620597248, + "active_bytes_peak": 123525784576, + "allocated_bytes_current": 84620597248, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6802647040, + "inactive_split_bytes_peak": 12487894016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 125044785152, + "reserved_bytes_peak": 125044785152, + "segment_count_current": 2906, + "segment_count_peak": 2906, + "timestamp_utc": "2026-08-26T18:58:37.862552Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_current_delta": 100397056, + "inactive_split_bytes_peak": 16642824192, + "inactive_split_bytes_peak_delta": 4154930176, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_current_delta": 21422407680, + "reserved_bytes_peak": 146467192832, + "reserved_bytes_peak_delta": 21422407680, + "segment_count_current": 3359, + "segment_count_current_delta": 453, + "segment_count_peak": 3359, + "segment_count_peak_delta": 453, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T185841332667Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.539937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185841332667Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185841332667Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46a3e74607acda9fe66a9ff9207e46c7a5b64327 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T185841332667Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:258e09ac62605e932a4921348fbfa838b67cb0a91db217dca5abde95e88cfe20 +size 11669613 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185841985800Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185841985800Z.json new file mode 100644 index 0000000000000000000000000000000000000000..443b82915b97b93fff03cd7760621b87ebb44b7d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185841985800Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637640704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130120022528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 130120022528, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_peak": 16642824192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_peak": 146467192832, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:41.985316Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637640704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123525784576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 130120022528, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_peak": 16642824192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_peak": 146467192832, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:41.569295Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16642824192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146467192832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3359, + "segment_count_current_delta": 0, + "segment_count_peak": 3359, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T185841985800Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.011387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185841985800Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185841985800Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87fedcc4a991972adc31faf0ff2d8535741983bf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T185841985800Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b843447bcbeb44968f6abadc44915475e304c94f4311b93be8992210e5da905 +size 1037479 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185842414105Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185842414105Z.json new file mode 100644 index 0000000000000000000000000000000000000000..238e6adf5898f1aad35dd276bd49bc19f9c5ade5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185842414105Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84637640704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130120022528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 130120022528, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_peak": 16642824192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_peak": 146467192832, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:42.413691Z", + "world_size": 8 + }, + "allocated_bytes_current": 84637640704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123525784576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84637640704, + "active_bytes_peak": 130120022528, + "allocated_bytes_current": 84637640704, + "allocated_bytes_peak": 123525784576, + "device": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_peak": 16642824192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_peak": 146467192832, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:42.013147Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6903044096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16642824192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 146467192832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146467192832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3359, + "segment_count_current_delta": 0, + "segment_count_peak": 3359, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T185842414105Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.438632Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185842414105Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185842414105Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..105bd31a9aa6f9638c337fee96896c5c48db99c9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T185842414105Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a58a48724d0323e07eca4878972232fd2b70c74ffcc6b14482fbda5adeaf4b2e +size 1037510 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185834760987Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185834760987Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f2c65ccbd41b981dacdbc0b209343a93662b4fd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185834760987Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84634637824, + "active_bytes_current_delta": 41081292800, + "active_bytes_peak": 92798320640, + "active_bytes_peak_delta": 49238684160, + "after": { + "active_bytes_current": 84634637824, + "active_bytes_peak": 92798320640, + "allocated_bytes_current": 84634637824, + "allocated_bytes_peak": 92798320640, + "device": 1, + "inactive_split_bytes_current": 7088499200, + "inactive_split_bytes_peak": 12726406144, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 94220845056, + "reserved_bytes_peak": 94220845056, + "segment_count_current": 2271, + "segment_count_peak": 2271, + "timestamp_utc": "2026-08-26T18:58:34.760489Z", + "world_size": 8 + }, + "allocated_bytes_current": 84634637824, + "allocated_bytes_current_delta": 41081292800, + "allocated_bytes_peak": 92798320640, + "allocated_bytes_peak_delta": 49238684160, + "before": { + "active_bytes_current": 43553345024, + "active_bytes_peak": 43559636480, + "allocated_bytes_current": 43553345024, + "allocated_bytes_peak": 43559636480, + "device": 1, + "inactive_split_bytes_current": 1254404608, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:57:56.845752Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7088499200, + "inactive_split_bytes_current_delta": 5834094592, + "inactive_split_bytes_peak": 12726406144, + "inactive_split_bytes_peak_delta": 11113493504, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 94220845056, + "reserved_bytes_current_delta": 49396318208, + "reserved_bytes_peak": 94220845056, + "reserved_bytes_peak_delta": 49396318208, + "segment_count_current": 2271, + "segment_count_current_delta": 1381, + "segment_count_peak": 2271, + "segment_count_peak_delta": 1381, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T185834760987Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.958430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185834760987Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185834760987Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3b7745eb0faab84b1c23357b14a0e8c5b0eaa53 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T185834760987Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51ed57a1c70ef38c71960eb92311642ce5474b6f3a0674bc14bdd8421e8851fe +size 9755367 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185836224584Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185836224584Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9f2f6ac9c5ed5317afdaa92c00705370e95ef6b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185836224584Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84634637824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123523190784, + "active_bytes_peak_delta": 30724870144, + "after": { + "active_bytes_current": 84634637824, + "active_bytes_peak": 123523190784, + "allocated_bytes_current": 84634637824, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 6989933056, + "inactive_split_bytes_peak": 12726406144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 124944121856, + "reserved_bytes_peak": 124944121856, + "segment_count_current": 2912, + "segment_count_peak": 2912, + "timestamp_utc": "2026-08-26T18:58:36.224136Z", + "world_size": 8 + }, + "allocated_bytes_current": 84634637824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123523190784, + "allocated_bytes_peak_delta": 30724870144, + "before": { + "active_bytes_current": 84634637824, + "active_bytes_peak": 92798320640, + "allocated_bytes_current": 84634637824, + "allocated_bytes_peak": 92798320640, + "device": 1, + "inactive_split_bytes_current": 7088499200, + "inactive_split_bytes_peak": 12726406144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 94220845056, + "reserved_bytes_peak": 94220845056, + "segment_count_current": 2271, + "segment_count_peak": 2271, + "timestamp_utc": "2026-08-26T18:58:34.998003Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6989933056, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12726406144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 124944121856, + "reserved_bytes_current_delta": 30723276800, + "reserved_bytes_peak": 124944121856, + "reserved_bytes_peak_delta": 30723276800, + "segment_count_current": 2912, + "segment_count_current_delta": 641, + "segment_count_peak": 2912, + "segment_count_peak_delta": 641, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T185836224584Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.417305Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185836224584Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185836224584Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51cf18d1c5aab64040c2d76df7e753c7463663cf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T185836224584Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f53e2f8fdfc455adc0b0237576f9c1526506fa483195206aa2a90267d8b32c19 +size 9999563 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185837639727Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185837639727Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca896a069845a7b1d8603ac4f7f83bf1acb741fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185837639727Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84634637824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123968344064, + "active_bytes_peak_delta": 445153280, + "after": { + "active_bytes_current": 84634637824, + "active_bytes_peak": 123968344064, + "allocated_bytes_current": 84634637824, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 7191259648, + "inactive_split_bytes_peak": 12726406144, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 125246111744, + "reserved_bytes_peak": 125246111744, + "segment_count_current": 2915, + "segment_count_peak": 2915, + "timestamp_utc": "2026-08-26T18:58:37.639298Z", + "world_size": 8 + }, + "allocated_bytes_current": 84634637824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123523190784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84634637824, + "active_bytes_peak": 123523190784, + "allocated_bytes_current": 84634637824, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 6989933056, + "inactive_split_bytes_peak": 12726406144, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 124944121856, + "reserved_bytes_peak": 124944121856, + "segment_count_current": 2912, + "segment_count_peak": 2912, + "timestamp_utc": "2026-08-26T18:58:36.443482Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7191259648, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 12726406144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 125246111744, + "reserved_bytes_current_delta": 301989888, + "reserved_bytes_peak": 125246111744, + "reserved_bytes_peak_delta": 301989888, + "segment_count_current": 2915, + "segment_count_current_delta": 3, + "segment_count_peak": 2915, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T185837639727Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.834332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185837639727Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185837639727Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a61e32dccdce531ea2c0a68fda509c8af75c223 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185837639727Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d99c8e53f05c76ced99bb3d66722725a9904ab2658fb726916bc0dbf545d57ad +size 10147501 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185841332693Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185841332693Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fe8ba694cf27ff3aebcd32234037061ee4c17588 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185841332693Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651681280, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130134063104, + "active_bytes_peak_delta": 6165719040, + "after": { + "active_bytes_current": 84651681280, + "active_bytes_peak": 130134063104, + "allocated_bytes_current": 84651681280, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_peak": 16830110208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 146869846016, + "reserved_bytes_peak": 146869846016, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:41.332176Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651681280, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123523190784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84634637824, + "active_bytes_peak": 123968344064, + "allocated_bytes_current": 84634637824, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 7191259648, + "inactive_split_bytes_peak": 12726406144, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 125246111744, + "reserved_bytes_peak": 125246111744, + "segment_count_current": 2915, + "segment_count_peak": 2915, + "timestamp_utc": "2026-08-26T18:58:37.861614Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_current_delta": -266240, + "inactive_split_bytes_peak": 16830110208, + "inactive_split_bytes_peak_delta": 4103704064, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 146869846016, + "reserved_bytes_current_delta": 21623734272, + "reserved_bytes_peak": 146869846016, + "reserved_bytes_peak_delta": 21623734272, + "segment_count_current": 3359, + "segment_count_current_delta": 444, + "segment_count_peak": 3359, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185841332693Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.542086Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185841332693Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185841332693Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..efa3d061b66acd73cc1362a1950b73b65dee0031 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185841332693Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfe34ded84953b7f2ec138d3884b96a876c74b1b9925682071aad469c173ef3f +size 13036588 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185841985800Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185841985800Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16bab8fd9aab14a115d0e991c6bf6b015f6a5de7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185841985800Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651681280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130134063104, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84651681280, + "active_bytes_peak": 130134063104, + "allocated_bytes_current": 84651681280, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_peak": 16830110208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 146869846016, + "reserved_bytes_peak": 146869846016, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:41.985328Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651681280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123523190784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84651681280, + "active_bytes_peak": 130134063104, + "allocated_bytes_current": 84651681280, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_peak": 16830110208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 146869846016, + "reserved_bytes_peak": 146869846016, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:41.571185Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16830110208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 146869846016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146869846016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3359, + "segment_count_current_delta": 0, + "segment_count_peak": 3359, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185841985800Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.011369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185841985800Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185841985800Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7c5f88c5aef0b4682513391addd2091e1ee6118 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185841985800Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f18dcadaa1c804b78bbdf8f63a28141f94732760eb2053b7db51a8d38d11f4fd +size 1142942 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185842414131Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185842414131Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc940cc78bf8825264cd80f1177b77026f02bcac --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185842414131Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84651681280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130134063104, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84651681280, + "active_bytes_peak": 130134063104, + "allocated_bytes_current": 84651681280, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_peak": 16830110208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 146869846016, + "reserved_bytes_peak": 146869846016, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:42.413716Z", + "world_size": 8 + }, + "allocated_bytes_current": 84651681280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123523190784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84651681280, + "active_bytes_peak": 130134063104, + "allocated_bytes_current": 84651681280, + "allocated_bytes_peak": 123523190784, + "device": 1, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_peak": 16830110208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 146869846016, + "reserved_bytes_peak": 146869846016, + "segment_count_current": 3359, + "segment_count_peak": 3359, + "timestamp_utc": "2026-08-26T18:58:42.013100Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7190993408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16830110208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 146869846016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146869846016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3359, + "segment_count_current_delta": 0, + "segment_count_peak": 3359, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185842414131Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.438469Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185842414131Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185842414131Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3fee6084812ac5fe436be00fe0d113642c0beda --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185842414131Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4851986bd0fce4b33371586a1c0585918cbabb8b23fdfd39cd670e83ae2044 +size 1142942 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185834760865Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185834760865Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28a64792a03a4cc6f47cc2d999ac6909f30e147a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185834760865Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 41083296256, + "active_bytes_peak": 92792233984, + "active_bytes_peak_delta": 49232399360, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 92792233984, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 92792233984, + "device": 2, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_peak": 13008471552, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 94023712768, + "reserved_bytes_peak": 94023712768, + "segment_count_current": 2257, + "segment_count_peak": 2257, + "timestamp_utc": "2026-08-26T18:58:34.760340Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 41083296256, + "allocated_bytes_peak": 92792233984, + "allocated_bytes_peak_delta": 49232399360, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 2, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:57:56.845720Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_current_delta": 6180218368, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 11395558912, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 94023712768, + "reserved_bytes_current_delta": 49188700160, + "reserved_bytes_peak": 94023712768, + "reserved_bytes_peak_delta": 49188700160, + "segment_count_current": 2257, + "segment_count_current_delta": 1374, + "segment_count_peak": 2257, + "segment_count_peak_delta": 1374, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185834760865Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.948784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185834760865Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185834760865Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d10eee42fb4ff3219c545bf4a5bd0a9f55b04e7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185834760865Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c152cc4683df8c5038d74aab635bd88d369405f3e7035c2d8ec51048a8d8aa1 +size 9737914 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185836224607Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185836224607Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60a346607a82df7e83b88cd7aa39e665a48fb966 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185836224607Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123536598528, + "active_bytes_peak_delta": 30744364544, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123536598528, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 124950413312, + "reserved_bytes_peak": 124950413312, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T18:58:36.224168Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 30744364544, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 92792233984, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 92792233984, + "device": 2, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_peak": 13008471552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 94023712768, + "reserved_bytes_peak": 94023712768, + "segment_count_current": 2257, + "segment_count_peak": 2257, + "timestamp_utc": "2026-08-26T18:58:34.986479Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 124950413312, + "reserved_bytes_current_delta": 30926700544, + "reserved_bytes_peak": 124950413312, + "reserved_bytes_peak_delta": 30926700544, + "segment_count_current": 2889, + "segment_count_current_delta": 632, + "segment_count_peak": 2889, + "segment_count_peak_delta": 632, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185836224607Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.417780Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185836224607Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185836224607Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f263f9fe3c09c0fcec0ba63fd636c908fc74f9dd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185836224607Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14b385d90df12227ad7899eae482275b3618e890e052fb2eef6c9639223a0abe +size 9785884 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185837639735Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185837639735Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cfbf854b2bf970465390b01765f2382954554fe2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185837639735Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123973920256, + "active_bytes_peak_delta": 437321728, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123973920256, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 125252403200, + "reserved_bytes_peak": 125252403200, + "segment_count_current": 2892, + "segment_count_peak": 2892, + "timestamp_utc": "2026-08-26T18:58:37.639324Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123536598528, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 124950413312, + "reserved_bytes_peak": 124950413312, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T18:58:36.445821Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 125252403200, + "reserved_bytes_current_delta": 301989888, + "reserved_bytes_peak": 125252403200, + "reserved_bytes_peak_delta": 301989888, + "segment_count_current": 2892, + "segment_count_current_delta": 3, + "segment_count_peak": 2892, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185837639735Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.834828Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185837639735Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185837639735Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5695093511dbc798db47d8aad176d81b28ff41a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185837639735Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a7a6e7b431fb6a7857e1565b8c05cff358960fa7ab64ae00b4ad7e6b3ffc25 +size 9666049 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185841332801Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185841332801Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f1e6d4e146f12a0998ffcf4912851ec3916ff2e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185841332801Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 6162344448, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17127801344, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.332269Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123973920256, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 125252403200, + "reserved_bytes_peak": 125252403200, + "segment_count_current": 2892, + "segment_count_peak": 2892, + "timestamp_utc": "2026-08-26T18:58:37.861605Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": -266240, + "inactive_split_bytes_peak": 17127801344, + "inactive_split_bytes_peak_delta": 4119329792, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 21967667200, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 21967667200, + "segment_count_current": 3354, + "segment_count_current_delta": 462, + "segment_count_peak": 3354, + "segment_count_peak_delta": 462, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185841332801Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.543910Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185841332801Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185841332801Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5630df1e74d928c8584c86b85d7336bfe3ef4012 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185841332801Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e18ab70212bd0503fbd3aa31eb2288fa896ad9ed204643eae78e95477208428e +size 11654798 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185841985919Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185841985919Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d966bec31ee02866a4f4b76ad108524652fd691 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185841985919Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17127801344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.985434Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17127801344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.573746Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17127801344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3354, + "segment_count_current_delta": 0, + "segment_count_peak": 3354, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185841985919Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.013329Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185841985919Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185841985919Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..595a85a726990b713f9350cb6c17c5fe92a8601b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185841985919Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d341a9bdf624d83270870636da44723969408530b370492b3e0ba4418a3dd25 +size 1024158 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185842414173Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185842414173Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c54e9e8ec69abc7d3ae463d5477ead53fc809a97 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185842414173Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17127801344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:42.413745Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 2, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17127801344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:42.015076Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17127801344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3354, + "segment_count_current_delta": 0, + "segment_count_peak": 3354, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185842414173Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.438234Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185842414173Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185842414173Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62ffcfc935883cb9bcf2b9c98b28e5fe05b525c4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185842414173Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25ed3e02e75f70352097a550a6e1a2e21b580c308f517f76490aa72419c610ec +size 1024158 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185834761160Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185834761160Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee49028cb15d976af3f5169fb4d63930dada3935 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185834761160Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84639997952, + "active_bytes_current_delta": 41083175424, + "active_bytes_peak": 92799373824, + "active_bytes_peak_delta": 49236259840, + "after": { + "active_bytes_current": 84639997952, + "active_bytes_peak": 92799373824, + "allocated_bytes_current": 84639997952, + "allocated_bytes_peak": 92799373824, + "device": 3, + "inactive_split_bytes_current": 6770663424, + "inactive_split_bytes_peak": 12433521664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93931438080, + "reserved_bytes_peak": 93931438080, + "segment_count_current": 2267, + "segment_count_peak": 2267, + "timestamp_utc": "2026-08-26T18:58:34.760685Z", + "world_size": 8 + }, + "allocated_bytes_current": 84639997952, + "allocated_bytes_current_delta": 41083175424, + "allocated_bytes_peak": 92799373824, + "allocated_bytes_peak_delta": 49236259840, + "before": { + "active_bytes_current": 43556822528, + "active_bytes_peak": 43563113984, + "allocated_bytes_current": 43556822528, + "allocated_bytes_peak": 43563113984, + "device": 3, + "inactive_split_bytes_current": 1223664128, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T18:57:56.845576Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6770663424, + "inactive_split_bytes_current_delta": 5546999296, + "inactive_split_bytes_peak": 12433521664, + "inactive_split_bytes_peak_delta": 10820609024, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93931438080, + "reserved_bytes_current_delta": 49136271360, + "reserved_bytes_peak": 93931438080, + "reserved_bytes_peak_delta": 49136271360, + "segment_count_current": 2267, + "segment_count_current_delta": 1380, + "segment_count_peak": 2267, + "segment_count_peak_delta": 1380, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185834761160Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.955959Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185834761160Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185834761160Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3800264444374734ee133413bcec14633d03b409 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185834761160Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdf98970d5b94aafff7e163f1c9fbdeec617856a36f0e988ff952eab844475d9 +size 9754862 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185836224545Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185836224545Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24dd70be6cf0657dd35ea2ba56c1fe364ab6c542 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185836224545Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84639997952, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123489780736, + "active_bytes_peak_delta": 30690406912, + "after": { + "active_bytes_current": 84639997952, + "active_bytes_peak": 123489780736, + "allocated_bytes_current": 84639997952, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6875521024, + "inactive_split_bytes_peak": 12433521664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 124937830400, + "reserved_bytes_peak": 124937830400, + "segment_count_current": 2914, + "segment_count_peak": 2914, + "timestamp_utc": "2026-08-26T18:58:36.224121Z", + "world_size": 8 + }, + "allocated_bytes_current": 84639997952, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123489780736, + "allocated_bytes_peak_delta": 30690406912, + "before": { + "active_bytes_current": 84639997952, + "active_bytes_peak": 92799373824, + "allocated_bytes_current": 84639997952, + "allocated_bytes_peak": 92799373824, + "device": 3, + "inactive_split_bytes_current": 6770663424, + "inactive_split_bytes_peak": 12433521664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93931438080, + "reserved_bytes_peak": 93931438080, + "segment_count_current": 2267, + "segment_count_peak": 2267, + "timestamp_utc": "2026-08-26T18:58:34.986564Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6875521024, + "inactive_split_bytes_current_delta": 104857600, + "inactive_split_bytes_peak": 12433521664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 124937830400, + "reserved_bytes_current_delta": 31006392320, + "reserved_bytes_peak": 124937830400, + "reserved_bytes_peak_delta": 31006392320, + "segment_count_current": 2914, + "segment_count_current_delta": 647, + "segment_count_peak": 2914, + "segment_count_peak_delta": 647, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185836224545Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.405977Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185836224545Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185836224545Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4152c204f499e5fb4995ebdc2aac6dfce35a2bf5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185836224545Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffaf67e225dd6297f68cb71e6d1474a8479953d8da65214ff1659ba7ac445bf3 +size 10165112 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185837639692Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185837639692Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d1fad39fb149b91fe3fb5f1be5afdeb8753e957 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185837639692Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84639997952, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123930231808, + "active_bytes_peak_delta": 440451072, + "after": { + "active_bytes_current": 84639997952, + "active_bytes_peak": 123930231808, + "allocated_bytes_current": 84639997952, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6873423872, + "inactive_split_bytes_peak": 12433521664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 125239820288, + "reserved_bytes_peak": 125239820288, + "segment_count_current": 2917, + "segment_count_peak": 2917, + "timestamp_utc": "2026-08-26T18:58:37.639283Z", + "world_size": 8 + }, + "allocated_bytes_current": 84639997952, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123489780736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84639997952, + "active_bytes_peak": 123489780736, + "allocated_bytes_current": 84639997952, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6875521024, + "inactive_split_bytes_peak": 12433521664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 124937830400, + "reserved_bytes_peak": 124937830400, + "segment_count_current": 2914, + "segment_count_peak": 2914, + "timestamp_utc": "2026-08-26T18:58:36.428861Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6873423872, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 12433521664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 125239820288, + "reserved_bytes_current_delta": 301989888, + "reserved_bytes_peak": 125239820288, + "reserved_bytes_peak_delta": 301989888, + "segment_count_current": 2917, + "segment_count_current_delta": 3, + "segment_count_peak": 2917, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185837639692Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.822627Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185837639692Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185837639692Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22734265812268f60bf7865f6f834a3173f8cf27 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185837639692Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea578f46453a3fb69977d83bd226b2e3cf0f6a83f2a14f26d221d5107f277b01 +size 9680761 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185841332874Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185841332874Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87685a661c912b4995a9d3f92fd897f376978a95 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185841332874Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84657041408, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130139423232, + "active_bytes_peak_delta": 6209191424, + "after": { + "active_bytes_current": 84657041408, + "active_bytes_peak": 130139423232, + "allocated_bytes_current": 84657041408, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_peak": 16512274432, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 146557370368, + "reserved_bytes_peak": 146557370368, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:41.332268Z", + "world_size": 8 + }, + "allocated_bytes_current": 84657041408, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123489780736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84639997952, + "active_bytes_peak": 123930231808, + "allocated_bytes_current": 84639997952, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6873423872, + "inactive_split_bytes_peak": 12433521664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 125239820288, + "reserved_bytes_peak": 125239820288, + "segment_count_current": 2917, + "segment_count_peak": 2917, + "timestamp_utc": "2026-08-26T18:58:37.842251Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_current_delta": -266240, + "inactive_split_bytes_peak": 16512274432, + "inactive_split_bytes_peak_delta": 4078752768, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 146557370368, + "reserved_bytes_current_delta": 21317550080, + "reserved_bytes_peak": 146557370368, + "reserved_bytes_peak_delta": 21317550080, + "segment_count_current": 3361, + "segment_count_current_delta": 444, + "segment_count_peak": 3361, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185841332874Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.538260Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185841332874Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185841332874Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f8c8e38a3f35d875069398899b3c9ce5426190e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185841332874Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a725aeff16b97a319599e71b6f9a7c03b06ff6e1366a7ff8b2cab3f5e6404f0a +size 13036875 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185841985804Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185841985804Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc70d3ab9192f3c2e0f40bbcfa36db71c0448f03 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185841985804Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84657041408, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130139423232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84657041408, + "active_bytes_peak": 130139423232, + "allocated_bytes_current": 84657041408, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_peak": 16512274432, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 146557370368, + "reserved_bytes_peak": 146557370368, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:41.985344Z", + "world_size": 8 + }, + "allocated_bytes_current": 84657041408, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123489780736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84657041408, + "active_bytes_peak": 130139423232, + "allocated_bytes_current": 84657041408, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_peak": 16512274432, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 146557370368, + "reserved_bytes_peak": 146557370368, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:41.564604Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16512274432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 146557370368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146557370368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3361, + "segment_count_current_delta": 0, + "segment_count_peak": 3361, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185841985804Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.010413Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185841985804Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185841985804Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8405669a55eff682fd0ebc2e8e46ddd6f5142681 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185841985804Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d5a0b2eb98f9d4649711fb13e7449c8ad84d442c0129a4c24cfdff0d55b49c3 +size 1143240 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185842414112Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185842414112Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b7bbd3e71b7debaeec12de797eaf1a1a6b495e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185842414112Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84657041408, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130139423232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84657041408, + "active_bytes_peak": 130139423232, + "allocated_bytes_current": 84657041408, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_peak": 16512274432, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 146557370368, + "reserved_bytes_peak": 146557370368, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:42.413704Z", + "world_size": 8 + }, + "allocated_bytes_current": 84657041408, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123489780736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84657041408, + "active_bytes_peak": 130139423232, + "allocated_bytes_current": 84657041408, + "allocated_bytes_peak": 123489780736, + "device": 3, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_peak": 16512274432, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 146557370368, + "reserved_bytes_peak": 146557370368, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:42.012108Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6873157632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16512274432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 146557370368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146557370368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3361, + "segment_count_current_delta": 0, + "segment_count_peak": 3361, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185842414112Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.436344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185842414112Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185842414112Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88f9ac51c93baee891c083d4b07dd67c4ba5931e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185842414112Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cfd2df2cfdc532aa6559624aadcc1d0056792bb9a2342b6fd60f6a7b467be98 +size 1143240 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185834760855Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185834760855Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9178de2f59a007ffc019ab74d694c5b0f47cd37d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185834760855Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84647776256, + "active_bytes_current_delta": 41097098752, + "active_bytes_peak": 92787284480, + "active_bytes_peak_delta": 49230315520, + "after": { + "active_bytes_current": 84647776256, + "active_bytes_peak": 92787284480, + "allocated_bytes_current": 84647776256, + "allocated_bytes_peak": 92787284480, + "device": 4, + "inactive_split_bytes_current": 7427682304, + "inactive_split_bytes_peak": 12958519296, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93834969088, + "reserved_bytes_peak": 93834969088, + "segment_count_current": 2267, + "segment_count_peak": 2267, + "timestamp_utc": "2026-08-26T18:58:34.760326Z", + "world_size": 8 + }, + "allocated_bytes_current": 84647776256, + "allocated_bytes_current_delta": 41097098752, + "allocated_bytes_peak": 92787284480, + "allocated_bytes_peak_delta": 49230315520, + "before": { + "active_bytes_current": 43550677504, + "active_bytes_peak": 43556968960, + "allocated_bytes_current": 43550677504, + "allocated_bytes_peak": 43556968960, + "device": 4, + "inactive_split_bytes_current": 1252877824, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T18:57:56.845875Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7427682304, + "inactive_split_bytes_current_delta": 6174804480, + "inactive_split_bytes_peak": 12958519296, + "inactive_split_bytes_peak_delta": 11345606656, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93834969088, + "reserved_bytes_current_delta": 49014636544, + "reserved_bytes_peak": 93834969088, + "reserved_bytes_peak_delta": 49014636544, + "segment_count_current": 2267, + "segment_count_current_delta": 1378, + "segment_count_peak": 2267, + "segment_count_peak_delta": 1378, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185834760855Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.953977Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185834760855Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185834760855Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..157c8c76305a895754fc9b5dabd6902d4bf80bbe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185834760855Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7894d50cc97cfd5459cef7c11c2adf4f2acde529f4478b8c69239402e75aa2e +size 9754780 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185836224634Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185836224634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..097c14f80ebc8246d45f9feb61329e4a0f29cfdc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185836224634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84647776256, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123552337920, + "active_bytes_peak_delta": 30765053440, + "after": { + "active_bytes_current": 84647776256, + "active_bytes_peak": 123552337920, + "allocated_bytes_current": 84647776256, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7329116160, + "inactive_split_bytes_peak": 12958519296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 124579217408, + "reserved_bytes_peak": 124579217408, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T18:58:36.224166Z", + "world_size": 8 + }, + "allocated_bytes_current": 84647776256, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123552337920, + "allocated_bytes_peak_delta": 30765053440, + "before": { + "active_bytes_current": 84647776256, + "active_bytes_peak": 92787284480, + "allocated_bytes_current": 84647776256, + "allocated_bytes_peak": 92787284480, + "device": 4, + "inactive_split_bytes_current": 7427682304, + "inactive_split_bytes_peak": 12958519296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93834969088, + "reserved_bytes_peak": 93834969088, + "segment_count_current": 2267, + "segment_count_peak": 2267, + "timestamp_utc": "2026-08-26T18:58:34.991981Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7329116160, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 12958519296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 124579217408, + "reserved_bytes_current_delta": 30744248320, + "reserved_bytes_peak": 124579217408, + "reserved_bytes_peak_delta": 30744248320, + "segment_count_current": 2909, + "segment_count_current_delta": 642, + "segment_count_peak": 2909, + "segment_count_peak_delta": 642, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185836224634Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.429339Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185836224634Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185836224634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d01f21914926ada61a8e414341f4efc9c6c2e9f6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185836224634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6886b3f0a54e2618b35fef7a4a85cdbe0bdc12cd0d1c751f9f09bdf1eb4939a8 +size 9959306 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185837639693Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185837639693Z.json new file mode 100644 index 0000000000000000000000000000000000000000..22e706cb8a881dbe817737c5776f1ab34bc8f938 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185837639693Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84647776256, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123552337920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84647776256, + "active_bytes_peak": 123552337920, + "allocated_bytes_current": 84647776256, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7429779456, + "inactive_split_bytes_peak": 12958519296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 124780544000, + "reserved_bytes_peak": 124780544000, + "segment_count_current": 2911, + "segment_count_peak": 2911, + "timestamp_utc": "2026-08-26T18:58:37.639262Z", + "world_size": 8 + }, + "allocated_bytes_current": 84647776256, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123552337920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84647776256, + "active_bytes_peak": 123552337920, + "allocated_bytes_current": 84647776256, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7329116160, + "inactive_split_bytes_peak": 12958519296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 124579217408, + "reserved_bytes_peak": 124579217408, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T18:58:36.457141Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7429779456, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 12958519296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 124780544000, + "reserved_bytes_current_delta": 201326592, + "reserved_bytes_peak": 124780544000, + "reserved_bytes_peak_delta": 201326592, + "segment_count_current": 2911, + "segment_count_current_delta": 2, + "segment_count_peak": 2911, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185837639693Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.834735Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185837639693Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185837639693Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2866bc2286bce57841f990fac218ece28ebf6ea9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185837639693Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a03f8f9c21f67d6f5f035847b0ead7e2c3ca7f53fb330046023577cecafa02c4 +size 9700823 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185841332732Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185841332732Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca59cf56655144f6113e982b7bfd22dfa4bad1da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185841332732Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84664819712, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130147201536, + "active_bytes_peak_delta": 6594863616, + "after": { + "active_bytes_current": 84664819712, + "active_bytes_peak": 130147201536, + "allocated_bytes_current": 84664819712, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_peak": 17269956608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 147121504256, + "reserved_bytes_peak": 147121504256, + "segment_count_current": 3360, + "segment_count_peak": 3360, + "timestamp_utc": "2026-08-26T18:58:41.332205Z", + "world_size": 8 + }, + "allocated_bytes_current": 84664819712, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123552337920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84647776256, + "active_bytes_peak": 123552337920, + "allocated_bytes_current": 84647776256, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7429779456, + "inactive_split_bytes_peak": 12958519296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 124780544000, + "reserved_bytes_peak": 124780544000, + "segment_count_current": 2911, + "segment_count_peak": 2911, + "timestamp_utc": "2026-08-26T18:58:37.862065Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_current_delta": 100397056, + "inactive_split_bytes_peak": 17269956608, + "inactive_split_bytes_peak_delta": 4311437312, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 147121504256, + "reserved_bytes_current_delta": 22340960256, + "reserved_bytes_peak": 147121504256, + "reserved_bytes_peak_delta": 22340960256, + "segment_count_current": 3360, + "segment_count_current_delta": 449, + "segment_count_peak": 3360, + "segment_count_peak_delta": 449, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185841332732Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.544471Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185841332732Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185841332732Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f179cb211ec639e443e99ea735208c1bf50a1b82 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185841332732Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df454927f2814bbd9699769b965bba25ed8cc8fbec739d1e8a7a0a450664f7eb +size 13037908 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185841985819Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185841985819Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94babd5427124479d022b4a94091b093738235da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185841985819Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84664819712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130147201536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84664819712, + "active_bytes_peak": 130147201536, + "allocated_bytes_current": 84664819712, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_peak": 17269956608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 147121504256, + "reserved_bytes_peak": 147121504256, + "segment_count_current": 3360, + "segment_count_peak": 3360, + "timestamp_utc": "2026-08-26T18:58:41.985345Z", + "world_size": 8 + }, + "allocated_bytes_current": 84664819712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123552337920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84664819712, + "active_bytes_peak": 130147201536, + "allocated_bytes_current": 84664819712, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_peak": 17269956608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 147121504256, + "reserved_bytes_peak": 147121504256, + "segment_count_current": 3360, + "segment_count_peak": 3360, + "timestamp_utc": "2026-08-26T18:58:41.573881Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17269956608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 147121504256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147121504256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3360, + "segment_count_current_delta": 0, + "segment_count_peak": 3360, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185841985819Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.011401Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185841985819Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185841985819Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb7840c13be483817b49674de00d70330468eba2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185841985819Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b7ead5ae22772145763d61d1ebea9a6ac1fe47da504ebb5ed211a7ed03c133 +size 1145117 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185842414119Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185842414119Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5513b2dec9f33a06b68d51fe8de9d86d553270a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185842414119Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84664819712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130147201536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84664819712, + "active_bytes_peak": 130147201536, + "allocated_bytes_current": 84664819712, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_peak": 17269956608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 147121504256, + "reserved_bytes_peak": 147121504256, + "segment_count_current": 3360, + "segment_count_peak": 3360, + "timestamp_utc": "2026-08-26T18:58:42.413705Z", + "world_size": 8 + }, + "allocated_bytes_current": 84664819712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123552337920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84664819712, + "active_bytes_peak": 130147201536, + "allocated_bytes_current": 84664819712, + "allocated_bytes_peak": 123552337920, + "device": 4, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_peak": 17269956608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 147121504256, + "reserved_bytes_peak": 147121504256, + "segment_count_current": 3360, + "segment_count_peak": 3360, + "timestamp_utc": "2026-08-26T18:58:42.013145Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7530176512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17269956608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 147121504256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147121504256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3360, + "segment_count_current_delta": 0, + "segment_count_peak": 3360, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185842414119Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.438779Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185842414119Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185842414119Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37b96469294cc1f95ee6acda9f9f35ccc46e9d50 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185842414119Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d398923e030c889b1f9ecae9ddeb4726939355ab1c7f2429c4bbb620e2fbfbf +size 1145117 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185834760866Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185834760866Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0c42fa959307d3c74837517e1fc4fd569f01eec --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185834760866Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 41083296256, + "active_bytes_peak": 92792233984, + "active_bytes_peak_delta": 49232399360, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 92792233984, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 92792233984, + "device": 5, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_peak": 13008471552, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93923049472, + "reserved_bytes_peak": 93923049472, + "segment_count_current": 2256, + "segment_count_peak": 2256, + "timestamp_utc": "2026-08-26T18:58:34.760387Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 41083296256, + "allocated_bytes_peak": 92792233984, + "allocated_bytes_peak_delta": 49232399360, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 5, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:57:56.845722Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_current_delta": 6180218368, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 11395558912, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93923049472, + "reserved_bytes_current_delta": 49088036864, + "reserved_bytes_peak": 93923049472, + "reserved_bytes_peak_delta": 49088036864, + "segment_count_current": 2256, + "segment_count_current_delta": 1373, + "segment_count_peak": 2256, + "segment_count_peak_delta": 1373, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185834760866Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.965353Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185834760866Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185834760866Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c063582b07adf5e838b8d80faf2955a36387b0c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185834760866Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e64557c5c07e77047a1f2ad099cbc800476baa9999410cb412a910c4ef1d88c8 +size 9736743 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185836224587Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185836224587Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da585681b2823ecab694dcfe9810d3ff56e63a4c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185836224587Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123536598528, + "active_bytes_peak_delta": 30744364544, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123536598528, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7337955840, + "inactive_split_bytes_peak": 13008471552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 124648423424, + "reserved_bytes_peak": 124648423424, + "segment_count_current": 2886, + "segment_count_peak": 2886, + "timestamp_utc": "2026-08-26T18:58:36.224145Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 30744364544, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 92792233984, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 92792233984, + "device": 5, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_peak": 13008471552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93923049472, + "reserved_bytes_peak": 93923049472, + "segment_count_current": 2256, + "segment_count_peak": 2256, + "timestamp_utc": "2026-08-26T18:58:35.002542Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7337955840, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 124648423424, + "reserved_bytes_current_delta": 30725373952, + "reserved_bytes_peak": 124648423424, + "reserved_bytes_peak_delta": 30725373952, + "segment_count_current": 2886, + "segment_count_current_delta": 630, + "segment_count_peak": 2886, + "segment_count_peak_delta": 630, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185836224587Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.419487Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185836224587Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185836224587Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ab209a97491e2eca1787c51f64391d3de58fc72 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185836224587Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1dc3a06ddeb5b3a6ba24bf7e7ac2cf1dea98d0dac70e38b2c5e43969b2f6234 +size 9499131 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185837639746Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185837639746Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ba2ec9f364631668671018f3acf89993ff9a841 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185837639746Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123973920256, + "active_bytes_peak_delta": 437321728, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123973920256, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 125051076608, + "reserved_bytes_peak": 125051076608, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T18:58:37.639323Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123536598528, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7337955840, + "inactive_split_bytes_peak": 13008471552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 124648423424, + "reserved_bytes_peak": 124648423424, + "segment_count_current": 2886, + "segment_count_peak": 2886, + "timestamp_utc": "2026-08-26T18:58:36.445305Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_current_delta": 201326592, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 125051076608, + "reserved_bytes_current_delta": 402653184, + "reserved_bytes_peak": 125051076608, + "reserved_bytes_peak_delta": 402653184, + "segment_count_current": 2890, + "segment_count_current_delta": 4, + "segment_count_peak": 2890, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185837639746Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.843691Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185837639746Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185837639746Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3731f963f5aba09f16e78078f3bd499b54588f38 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185837639746Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c516052573ec55feaf699853374753f290324bb85d97fd2604b3d4891831a112 +size 9669877 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185841332676Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185841332676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..052adcfe63b06773fa73abe324758d191b5b138e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185841332676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 6162344448, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.332157Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123973920256, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 125051076608, + "reserved_bytes_peak": 125051076608, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T18:58:37.870482Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": -266240, + "inactive_split_bytes_peak": 17178132992, + "inactive_split_bytes_peak_delta": 4169661440, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 22168993792, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 22168993792, + "segment_count_current": 3354, + "segment_count_current_delta": 464, + "segment_count_peak": 3354, + "segment_count_peak_delta": 464, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185841332676Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.540246Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185841332676Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185841332676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a4dba03be49857a8dc7cba50204704ce0ef4748 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185841332676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59eac8f3eefdb7d63ed5fd5f0a9ab793405711e853264072532f72ace8463aed +size 11655719 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185841985807Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185841985807Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e575fd0a5830226f21500d8aaeb3716ec9dee33 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185841985807Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.985336Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.568775Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17178132992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3354, + "segment_count_current_delta": 0, + "segment_count_peak": 3354, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185841985807Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.011356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185841985807Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185841985807Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2fd746e2c4de1feec00c0246de4308808995236 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185841985807Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2e10ac0076df33db4d55eca10cacc7a13c3c931b3d3ca2a037fb4d5bf74e3ea +size 1024406 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185842414186Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185842414186Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb2225e2f98ac5eddef974113d39887a17cc649b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185842414186Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:42.413767Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 5, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:42.013078Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17178132992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3354, + "segment_count_current_delta": 0, + "segment_count_peak": 3354, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185842414186Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.438288Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185842414186Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185842414186Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12fb666cfe4aa41b97a0128a46cfd93179b5bd51 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185842414186Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cffc638756d17fffd1e98b20a96ccf7b0982c72d794ccf26b360d3bd03c05b0e +size 1024406 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185834760865Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185834760865Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1781a3f42ec158a959eb8b71080aa9afd65f817d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185834760865Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84631882752, + "active_bytes_current_delta": 41078472192, + "active_bytes_peak": 92800849408, + "active_bytes_peak_delta": 49241147392, + "after": { + "active_bytes_current": 84631882752, + "active_bytes_peak": 92800849408, + "allocated_bytes_current": 84631882752, + "allocated_bytes_peak": 92800849408, + "device": 6, + "inactive_split_bytes_current": 6598423552, + "inactive_split_bytes_peak": 12306530304, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 94019518464, + "reserved_bytes_peak": 94019518464, + "segment_count_current": 2269, + "segment_count_peak": 2269, + "timestamp_utc": "2026-08-26T18:58:34.760363Z", + "world_size": 8 + }, + "allocated_bytes_current": 84631882752, + "allocated_bytes_current_delta": 41078472192, + "allocated_bytes_peak": 92800849408, + "allocated_bytes_peak_delta": 49241147392, + "before": { + "active_bytes_current": 43553410560, + "active_bytes_peak": 43559702016, + "allocated_bytes_current": 43553410560, + "allocated_bytes_peak": 43559702016, + "device": 6, + "inactive_split_bytes_current": 1256436224, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:57:56.845633Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6598423552, + "inactive_split_bytes_current_delta": 5341987328, + "inactive_split_bytes_peak": 12306530304, + "inactive_split_bytes_peak_delta": 10693617664, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 94019518464, + "reserved_bytes_current_delta": 49194991616, + "reserved_bytes_peak": 94019518464, + "reserved_bytes_peak_delta": 49194991616, + "segment_count_current": 2269, + "segment_count_current_delta": 1379, + "segment_count_peak": 2269, + "segment_count_peak_delta": 1379, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185834760865Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.954474Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185834760865Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185834760865Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0205e33a70cccba14a0955f704ef99866d0fa430 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185834760865Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a66e72e64024657a77ceda9b5abe00a14c1f3b611efa6d80fb72a0df099e1f +size 9756268 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185836224600Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185836224600Z.json new file mode 100644 index 0000000000000000000000000000000000000000..034146fdc62916c74f5704f66e350950558f8af3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185836224600Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84631882752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123495345664, + "active_bytes_peak_delta": 30694496256, + "after": { + "active_bytes_current": 84631882752, + "active_bytes_peak": 123495345664, + "allocated_bytes_current": 84631882752, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6501954560, + "inactive_split_bytes_peak": 12306530304, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 124702949376, + "reserved_bytes_peak": 124702949376, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T18:58:36.224160Z", + "world_size": 8 + }, + "allocated_bytes_current": 84631882752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123495345664, + "allocated_bytes_peak_delta": 30694496256, + "before": { + "active_bytes_current": 84631882752, + "active_bytes_peak": 92800849408, + "allocated_bytes_current": 84631882752, + "allocated_bytes_peak": 92800849408, + "device": 6, + "inactive_split_bytes_current": 6598423552, + "inactive_split_bytes_peak": 12306530304, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 94019518464, + "reserved_bytes_peak": 94019518464, + "segment_count_current": 2269, + "segment_count_peak": 2269, + "timestamp_utc": "2026-08-26T18:58:34.992496Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6501954560, + "inactive_split_bytes_current_delta": -96468992, + "inactive_split_bytes_peak": 12306530304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 124702949376, + "reserved_bytes_current_delta": 30683430912, + "reserved_bytes_peak": 124702949376, + "reserved_bytes_peak_delta": 30683430912, + "segment_count_current": 2909, + "segment_count_current_delta": 640, + "segment_count_peak": 2909, + "segment_count_peak_delta": 640, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185836224600Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.422497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185836224600Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185836224600Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..21db472b5cb9b575eaa3d237ab67fa07a3e1488f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185836224600Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25a56dc3be4e4f9ae3c1f7917a676bd54387f7028f6e328a8513b9d1a4e12844 +size 10000339 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185837639698Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185837639698Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e7a9bd7fdc7c54b8086998cd3b27927e60ebcc7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185837639698Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84631882752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123942088192, + "active_bytes_peak_delta": 446742528, + "after": { + "active_bytes_current": 84631882752, + "active_bytes_peak": 123942088192, + "allocated_bytes_current": 84631882752, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6701184000, + "inactive_split_bytes_peak": 12306530304, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125004939264, + "reserved_bytes_peak": 125004939264, + "segment_count_current": 2912, + "segment_count_peak": 2912, + "timestamp_utc": "2026-08-26T18:58:37.639284Z", + "world_size": 8 + }, + "allocated_bytes_current": 84631882752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123495345664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84631882752, + "active_bytes_peak": 123495345664, + "allocated_bytes_current": 84631882752, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6501954560, + "inactive_split_bytes_peak": 12306530304, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 124702949376, + "reserved_bytes_peak": 124702949376, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T18:58:36.451330Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6701184000, + "inactive_split_bytes_current_delta": 199229440, + "inactive_split_bytes_peak": 12306530304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125004939264, + "reserved_bytes_current_delta": 301989888, + "reserved_bytes_peak": 125004939264, + "reserved_bytes_peak_delta": 301989888, + "segment_count_current": 2912, + "segment_count_current_delta": 3, + "segment_count_peak": 2912, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185837639698Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.839897Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185837639698Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185837639698Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..339ad558cebf9981073db9a42e329f179fffa477 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185837639698Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d942f1821bb0bde7dcfbbb56d4fda04097823aa42f0169d7c7f37b45ee222a6 +size 10148572 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185841332719Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185841332719Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a88be63eeb130299377a97035adc7e16d57ef17 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185841332719Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84648926208, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130131308032, + "active_bytes_peak_delta": 6189219840, + "after": { + "active_bytes_current": 84648926208, + "active_bytes_peak": 130131308032, + "allocated_bytes_current": 84648926208, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_peak": 16340034560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 146377015296, + "reserved_bytes_peak": 146377015296, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:41.332178Z", + "world_size": 8 + }, + "allocated_bytes_current": 84648926208, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123495345664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84631882752, + "active_bytes_peak": 123942088192, + "allocated_bytes_current": 84631882752, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6701184000, + "inactive_split_bytes_peak": 12306530304, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 125004939264, + "reserved_bytes_peak": 125004939264, + "segment_count_current": 2912, + "segment_count_peak": 2912, + "timestamp_utc": "2026-08-26T18:58:37.866369Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_current_delta": -266240, + "inactive_split_bytes_peak": 16340034560, + "inactive_split_bytes_peak_delta": 4033504256, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 146377015296, + "reserved_bytes_current_delta": 21372076032, + "reserved_bytes_peak": 146377015296, + "reserved_bytes_peak_delta": 21372076032, + "segment_count_current": 3361, + "segment_count_current_delta": 449, + "segment_count_peak": 3361, + "segment_count_peak_delta": 449, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185841332719Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.548698Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185841332719Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185841332719Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f212a6798683f169e8dd9738f6b3ca987b7c82e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185841332719Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:724f779efffc16c97233eca6162fa547f82e5b654c3e88f3724d965885afb720 +size 13038125 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185841985819Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185841985819Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3936b0e1aed6f11e0efac22bb1401fca7ac562fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185841985819Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84648926208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130131308032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84648926208, + "active_bytes_peak": 130131308032, + "allocated_bytes_current": 84648926208, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_peak": 16340034560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 146377015296, + "reserved_bytes_peak": 146377015296, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:41.985361Z", + "world_size": 8 + }, + "allocated_bytes_current": 84648926208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123495345664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84648926208, + "active_bytes_peak": 130131308032, + "allocated_bytes_current": 84648926208, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_peak": 16340034560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 146377015296, + "reserved_bytes_peak": 146377015296, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:41.577637Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16340034560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 146377015296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146377015296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3361, + "segment_count_current_delta": 0, + "segment_count_peak": 3361, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185841985819Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.011573Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185841985819Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185841985819Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7977271b21a213e168b3a4e785395827c8ea7722 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185841985819Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a7c92151e18a6dbe92cad65677a82c3e4398a0e0a12a1d9962d9384f90207c5 +size 1144655 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185842414151Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185842414151Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31d9fa9559e69714519620ecdbd2b21ccb097b43 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185842414151Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84648926208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130131308032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84648926208, + "active_bytes_peak": 130131308032, + "allocated_bytes_current": 84648926208, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_peak": 16340034560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 146377015296, + "reserved_bytes_peak": 146377015296, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:42.413736Z", + "world_size": 8 + }, + "allocated_bytes_current": 84648926208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123495345664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84648926208, + "active_bytes_peak": 130131308032, + "allocated_bytes_current": 84648926208, + "allocated_bytes_peak": 123495345664, + "device": 6, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_peak": 16340034560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 146377015296, + "reserved_bytes_peak": 146377015296, + "segment_count_current": 3361, + "segment_count_peak": 3361, + "timestamp_utc": "2026-08-26T18:58:42.013302Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6700917760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16340034560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 146377015296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 146377015296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3361, + "segment_count_current_delta": 0, + "segment_count_peak": 3361, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185842414151Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.438848Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185842414151Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185842414151Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70ee05a2c9862a1f8a30fc0e9b0085f5fadfa200 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185842414151Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13fc239169d1db60a32650fc824418aac457e66b53bb7ed8e036179661175952 +size 1144655 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185834761128Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185834761128Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cafad06831b9e0cbc6e86b5861bf7cdb44df72d5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185834761128Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 41083296256, + "active_bytes_peak": 92792233984, + "active_bytes_peak_delta": 49232399360, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 92792233984, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 92792233984, + "device": 7, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_peak": 13008471552, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93822386176, + "reserved_bytes_peak": 93822386176, + "segment_count_current": 2255, + "segment_count_peak": 2255, + "timestamp_utc": "2026-08-26T18:58:34.760639Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 41083296256, + "allocated_bytes_peak": 92792233984, + "allocated_bytes_peak_delta": 49232399360, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 7, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:57:56.845715Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_current_delta": 6180218368, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 11395558912, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93822386176, + "reserved_bytes_current_delta": 48987373568, + "reserved_bytes_peak": 93822386176, + "reserved_bytes_peak_delta": 48987373568, + "segment_count_current": 2255, + "segment_count_current_delta": 1372, + "segment_count_peak": 2255, + "segment_count_peak_delta": 1372, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185834761128Z.pickle", + "timestamp_utc": "2026-08-26T18:58:34.952231Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185834761128Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185834761128Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f719e2fef3ca2cdb96342721da7dc43e4dfa223 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185834761128Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6cdbe961db7984690a8ee8863170f7f7c37b24648a577648d21f05ff115bc91 +size 9736565 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185836224821Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185836224821Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f651600f9751c094600ba146859464b8a15837a7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185836224821Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123536598528, + "active_bytes_peak_delta": 30744364544, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123536598528, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 124849750016, + "reserved_bytes_peak": 124849750016, + "segment_count_current": 2888, + "segment_count_peak": 2888, + "timestamp_utc": "2026-08-26T18:58:36.224321Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 30744364544, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 92792233984, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 92792233984, + "device": 7, + "inactive_split_bytes_current": 7436521984, + "inactive_split_bytes_peak": 13008471552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93822386176, + "reserved_bytes_peak": 93822386176, + "segment_count_current": 2255, + "segment_count_peak": 2255, + "timestamp_utc": "2026-08-26T18:58:34.983867Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 124849750016, + "reserved_bytes_current_delta": 31027363840, + "reserved_bytes_peak": 124849750016, + "reserved_bytes_peak_delta": 31027363840, + "segment_count_current": 2888, + "segment_count_current_delta": 633, + "segment_count_peak": 2888, + "segment_count_peak_delta": 633, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185836224821Z.pickle", + "timestamp_utc": "2026-08-26T18:58:36.409435Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185836224821Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185836224821Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1b57489b53ae8101f71edd223be38db28acc8f3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185836224821Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43db307a78f2099eb8b7b1623e7353adb3e0454594909671d98faa42a3e6ca4d +size 9856584 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185837639945Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185837639945Z.json new file mode 100644 index 0000000000000000000000000000000000000000..220bf628ece3969f499e7d07de121da35d3bfcfb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185837639945Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84636839424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123822925312, + "active_bytes_peak_delta": 286326784, + "after": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123822925312, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 124950413312, + "reserved_bytes_peak": 124950413312, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T18:58:37.639484Z", + "world_size": 8 + }, + "allocated_bytes_current": 84636839424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123536598528, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 124849750016, + "reserved_bytes_peak": 124849750016, + "segment_count_current": 2888, + "segment_count_peak": 2888, + "timestamp_utc": "2026-08-26T18:58:36.431816Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13008471552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 124950413312, + "reserved_bytes_current_delta": 100663296, + "reserved_bytes_peak": 124950413312, + "reserved_bytes_peak_delta": 100663296, + "segment_count_current": 2889, + "segment_count_current_delta": 1, + "segment_count_peak": 2889, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185837639945Z.pickle", + "timestamp_utc": "2026-08-26T18:58:37.824506Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185837639945Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185837639945Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35e96ce63425d4d4fcbb5abddb43a6c0ee97b1a9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185837639945Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53d3d924c95348e3e65e3f746408d786d3b53fa29da78acb2e2170f37cc43276 +size 9665851 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185841332770Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185841332770Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea9d4cd1ec4c1963a700e5b4df80e489c9102c28 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185841332770Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 6313339392, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.332188Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84636839424, + "active_bytes_peak": 123822925312, + "allocated_bytes_current": 84636839424, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539282432, + "inactive_split_bytes_peak": 13008471552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 124950413312, + "reserved_bytes_peak": 124950413312, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T18:58:37.845931Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": -266240, + "inactive_split_bytes_peak": 17178132992, + "inactive_split_bytes_peak_delta": 4169661440, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 22269657088, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 22269657088, + "segment_count_current": 3354, + "segment_count_current_delta": 465, + "segment_count_peak": 3354, + "segment_count_peak_delta": 465, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185841332770Z.pickle", + "timestamp_utc": "2026-08-26T18:58:41.545641Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185841332770Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185841332770Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40b89b60d9556828ccad73e0488fd56ee65702b0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185841332770Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d060dc8e3f98348be6a4f3620099b1d5fa96a6f81c3a354d6767480841ca462 +size 11655033 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185841985863Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185841985863Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1a9c1a527eb3971ce8827352a20b2ad337732f6e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185841985863Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.985390Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:41.574463Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17178132992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3354, + "segment_count_current_delta": 0, + "segment_count_peak": 3354, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185841985863Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.012633Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185841985863Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185841985863Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb6450e4441b7a3a574148d8800884c785d4718e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185841985863Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f36ee6b86b343bf615c691dfbe2cc489d4f4becdac9a12434235e8e5a380734a +size 1024271 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185842414181Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185842414181Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fe39d75f3aaeae331cf6505831b16882f33f1b6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185842414181Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84653882880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130136264704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:42.413766Z", + "world_size": 8 + }, + "allocated_bytes_current": 84653882880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123536598528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84653882880, + "active_bytes_peak": 130136264704, + "allocated_bytes_current": 84653882880, + "allocated_bytes_peak": 123536598528, + "device": 7, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_peak": 17178132992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 147220070400, + "reserved_bytes_peak": 147220070400, + "segment_count_current": 3354, + "segment_count_peak": 3354, + "timestamp_utc": "2026-08-26T18:58:42.014407Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7539016192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17178132992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 147220070400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147220070400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3354, + "segment_count_current_delta": 0, + "segment_count_peak": 3354, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185842414181Z.pickle", + "timestamp_utc": "2026-08-26T18:58:42.438921Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185842414181Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185842414181Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8721fb45e83db7061a10c6781febca73db3ffdbc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185842414181Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dcf5cc63e2e7bb84b6a8a3bdf4ccd151655a2b7bb71102bf6eea93225de0a5b +size 1024271 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..4a03b2350aba5ba1c1e42f1066b4532d5b04f0f8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..caa8f9ddeea15135e9f101c4e8f31f005d3c108e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log @@ -0,0 +1,2805 @@ +[18:57:30] WARN: better-memory-dump-support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 18:57:31.505000 585529 torch/distributed/run.py:766] +W0826 18:57:31.505000 585529 torch/distributed/run.py:766] ***************************************** +W0826 18:57:31.505000 585529 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:57:31.505000 585529 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 6 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank 77 is connected to 0 +7 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 235 is connected to 4 is connected to is connected to 7 is connected to 77 peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 77 + + +7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 3 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 18:57:49.653066532 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:57:49.666344039 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:57:49.666469422 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:57:49.667330229 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:57:49.667394770 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:57:49.667479698 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:57:49.667481675 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 18:57:49.847877685 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.617 seconds +time to initialize megatron (seconds): 6.838 +[after megatron is initialized] datetime: 2026-08-26 18:57:55 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.linear_qkv.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.k_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_qkv.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_proj.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.k_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.q_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:57:56 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 18:57:56 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (1100.67, 1314.92) + train/valid/test-data-iterators-setup ..........: (64.46, 68.38) +[before the start of training step] datetime: 2026-08-26 18:57:56 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 18:58:34] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 37917.0 | throughput per GPU (TFLOP/s/GPU): 2.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095255E+01 | loss scale: 1.0 | grad norm: 54.337 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 80700.490234375 | max allocated: 88506.0205078125 | reserved: 89970.0 | max reserved: 89970.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 18:58:36] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 1464.1 | throughput per GPU (TFLOP/s/GPU): 60.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095489E+01 | loss scale: 1.0 | grad norm: 56.502 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:58:37] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 1415.1 | throughput per GPU (TFLOP/s/GPU): 63.0 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008668E+01 | loss scale: 1.0 | grad norm: 180.249 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 18:58:41] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 3692.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.603720E+00 | loss scale: 1.0 | grad norm: 8.764 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:58:41] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 653.3 | throughput per GPU (TFLOP/s/GPU): 136.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.309318E+00 | loss scale: 1.0 | grad norm: 5.973 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:58:42] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 428.3 | throughput per GPU (TFLOP/s/GPU): 208.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.169638E+00 | loss scale: 1.0 | grad norm: 3.449 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:58:42 +[rank0]:[W826 18:58:43.846351734 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..a313a9661af8924282a46522f4a1a768466e6921 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,272 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 47 +2, 47 +3, 47 +4, 47 +5, 629 +6, 629 +7, 629 +0, 919 +1, 925 +2, 919 +3, 937 +4, 929 +5, 923 +6, 925 +7, 917 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 39169 +1, 38995 +2, 39197 +3, 38995 +4, 38995 +5, 39197 +6, 38995 +7, 39197 +0, 44989 +1, 44991 +2, 45001 +3, 44963 +4, 44951 +5, 45001 +6, 44991 +7, 45001 +0, 45895 +1, 45897 +2, 45907 +3, 45869 +4, 45893 +5, 45907 +6, 45897 +7, 45907 +0, 46917 +1, 46919 +2, 46929 +3, 46891 +4, 46915 +5, 46929 +6, 46919 +7, 46929 +0, 47013 +1, 47015 +2, 47025 +3, 46987 +4, 47011 +5, 47025 +6, 47015 +7, 47025 +0, 47179 +1, 47179 +2, 47205 +3, 47157 +4, 47191 +5, 47177 +6, 47181 +7, 47191 +0, 47877 +1, 47879 +2, 47889 +3, 47851 +4, 47875 +5, 47889 +6, 47879 +7, 47889 +0, 48011 +1, 48011 +2, 48023 +3, 47985 +4, 48007 +5, 48021 +6, 48011 +7, 48023 +0, 48551 +1, 48567 +2, 48563 +3, 48545 +4, 48543 +5, 48557 +6, 48527 +7, 48563 +0, 48817 +1, 48829 +2, 48825 +3, 48811 +4, 48805 +5, 48817 +6, 48791 +7, 48825 +0, 50629 +1, 50627 +2, 50623 +3, 50597 +4, 50221 +5, 50615 +6, 50569 +7, 50239 +0, 91927 +1, 91959 +2, 91923 +3, 91949 +4, 91957 +5, 91935 +6, 91935 +7, 91947 +0, 92563 +1, 92597 +2, 92557 +3, 92575 +4, 92591 +5, 92571 +6, 92557 +7, 92579 +0, 92563 +1, 92597 +2, 92557 +3, 92575 +4, 92591 +5, 92571 +6, 92557 +7, 92579 +0, 92563 +1, 92597 +2, 92557 +3, 92575 +4, 92591 +5, 92571 +6, 92557 +7, 92579 +0, 92563 +1, 92597 +2, 92557 +3, 92575 +4, 92591 +5, 92571 +6, 92557 +7, 92579 +0, 93089 +1, 93123 +2, 93057 +3, 93073 +4, 93083 +5, 93081 +6, 93083 +7, 93087 +0, 93635 +1, 93669 +2, 93629 +3, 93455 +4, 93663 +5, 93643 +6, 93149 +7, 93651 +0, 93787 +1, 93807 +2, 93797 +3, 93613 +4, 93833 +5, 93793 +6, 93591 +7, 93807 +0, 94599 +1, 94669 +2, 94607 +3, 94477 +4, 94621 +5, 94633 +6, 94451 +7, 94605 +0, 105127 +1, 105149 +2, 105303 +3, 104919 +4, 105165 +5, 105187 +6, 104913 +7, 105293 +0, 124489 +1, 124447 +2, 124481 +3, 124281 +4, 124481 +5, 124463 +6, 124251 +7, 124485 +0, 124489 +1, 124447 +2, 124481 +3, 124281 +4, 124481 +5, 124463 +6, 124251 +7, 124485 +0, 124489 +1, 124447 +2, 124481 +3, 124281 +4, 124481 +5, 124463 +6, 124251 +7, 124485 +0, 124489 +1, 124447 +2, 124481 +3, 124281 +4, 124481 +5, 124463 +6, 124251 +7, 124485 +0, 1223 +1, 124277 +2, 2506 +3, 906 +4, 906 +5, 906 +6, 906 +7, 906 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190001924146Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190001924146Z.json new file mode 100644 index 0000000000000000000000000000000000000000..529f81690f9f8c43154aea11a431e26592da5cfe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190001924146Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84709484032, + "active_bytes_current_delta": 41157881856, + "active_bytes_peak": 90923064320, + "active_bytes_peak_delta": 47365170688, + "after": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 90923064320, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 90923064320, + "device": 0, + "inactive_split_bytes_current": 4383824384, + "inactive_split_bytes_peak": 14330626560, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 92171927552, + "reserved_bytes_peak": 92171927552, + "segment_count_current": 2177, + "segment_count_peak": 2177, + "timestamp_utc": "2026-08-26T19:00:01.923662Z", + "world_size": 8 + }, + "allocated_bytes_current": 84709484032, + "allocated_bytes_current_delta": 41157881856, + "allocated_bytes_peak": 90923064320, + "allocated_bytes_peak_delta": 47365170688, + "before": { + "active_bytes_current": 43551602176, + "active_bytes_peak": 43557893632, + "allocated_bytes_current": 43551602176, + "allocated_bytes_peak": 43557893632, + "device": 0, + "inactive_split_bytes_current": 1256147456, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T18:59:17.789192Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4383824384, + "inactive_split_bytes_current_delta": 3127676928, + "inactive_split_bytes_peak": 14330626560, + "inactive_split_bytes_peak_delta": 12717713920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 92171927552, + "reserved_bytes_current_delta": 47349497856, + "reserved_bytes_peak": 92171927552, + "reserved_bytes_peak_delta": 47349497856, + "segment_count_current": 2177, + "segment_count_current_delta": 1292, + "segment_count_peak": 2177, + "segment_count_peak_delta": 1292, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190001924146Z.pickle", + "timestamp_utc": "2026-08-26T19:00:02.118796Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190001924146Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190001924146Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f2aebf69da9cb8e40aafdb38c021a05dcf028186 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190001924146Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dfc8bfe8e5384718a3795c7f9b7b5911ad4ba4483007afe09527ffaf4b4a067 +size 9424143 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190004404131Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190004404131Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f66e2c536a160373b1de4b56c76696e90576db6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190004404131Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84709484032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121757448192, + "active_bytes_peak_delta": 30834383872, + "after": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 121757448192, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 121757448192, + "device": 0, + "inactive_split_bytes_current": 4285258240, + "inactive_split_bytes_peak": 14330626560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_peak": 123029422080, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T19:00:04.403697Z", + "world_size": 8 + }, + "allocated_bytes_current": 84709484032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121757448192, + "allocated_bytes_peak_delta": 30834383872, + "before": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 90923064320, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 90923064320, + "device": 0, + "inactive_split_bytes_current": 4383824384, + "inactive_split_bytes_peak": 14330626560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 92171927552, + "reserved_bytes_peak": 92171927552, + "segment_count_current": 2177, + "segment_count_peak": 2177, + "timestamp_utc": "2026-08-26T19:00:02.155355Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4285258240, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14330626560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_current_delta": 30857494528, + "reserved_bytes_peak": 123029422080, + "reserved_bytes_peak_delta": 30857494528, + "segment_count_current": 2909, + "segment_count_current_delta": 732, + "segment_count_peak": 2909, + "segment_count_peak_delta": 732, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190004404131Z.pickle", + "timestamp_utc": "2026-08-26T19:00:04.601394Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190004404131Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190004404131Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e94e16bd5132f266be6faa0de920532c57c7c21 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190004404131Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03ab64fb92cd57f36c33df2ab2f114dc5a03c484727a8ab8c71bad2cabf7c029 +size 9862096 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190005872664Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190005872664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e889cc6835e8abb6c3b5fd1ef4f22c15b032dd4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190005872664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84709484032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121757448192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 121757448192, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 121757448192, + "device": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_peak": 14615533056, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_peak": 123029422080, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T19:00:05.872214Z", + "world_size": 8 + }, + "allocated_bytes_current": 84709484032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121757448192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 121757448192, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 121757448192, + "device": 0, + "inactive_split_bytes_current": 4285258240, + "inactive_split_bytes_peak": 14330626560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_peak": 123029422080, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T19:00:04.628230Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14615533056, + "inactive_split_bytes_peak_delta": 284906496, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123029422080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2909, + "segment_count_current_delta": 0, + "segment_count_peak": 2909, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190005872664Z.pickle", + "timestamp_utc": "2026-08-26T19:00:06.059903Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190005872664Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190005872664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..109b38d900d963f6cd1b5253892b79f04b8be119 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190005872664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd10e82651680453cd4fb81ff3aa3e243e5a02ff6c3a57415b8dd920f5a68ba4 +size 9834568 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190007540625Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190007540625Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b077ec48cb94c77ab4d4588c485f14c7962d645b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190007540625Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84709484032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121757448192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 121757448192, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 121757448192, + "device": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_peak": 15416847360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_peak": 123029422080, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T19:00:07.540178Z", + "world_size": 8 + }, + "allocated_bytes_current": 84709484032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121757448192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 121757448192, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 121757448192, + "device": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_peak": 14615533056, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_peak": 123029422080, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T19:00:06.084756Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15416847360, + "inactive_split_bytes_peak_delta": 801314304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123029422080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2909, + "segment_count_current_delta": 0, + "segment_count_peak": 2909, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190007540625Z.pickle", + "timestamp_utc": "2026-08-26T19:00:07.721995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190007540625Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190007540625Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d50dd8faaeb7942780b1f0d6e6140d01ea87fc7b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190007540625Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e37a372799baaa1e29315619cdc9021f79a94e69ca77da3026b2e99ed9915744 +size 9833363 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190008909179Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190008909179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6743a8d480b3a6d3a104ad467619007cef80298 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190008909179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84709484032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121757448192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 121757448192, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 121757448192, + "device": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_peak": 15583728128, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_peak": 123029422080, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T19:00:08.908739Z", + "world_size": 8 + }, + "allocated_bytes_current": 84709484032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121757448192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84709484032, + "active_bytes_peak": 121757448192, + "allocated_bytes_current": 84709484032, + "allocated_bytes_peak": 121757448192, + "device": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_peak": 15416847360, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_peak": 123029422080, + "segment_count_current": 2909, + "segment_count_peak": 2909, + "timestamp_utc": "2026-08-26T19:00:07.745817Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4385921536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15583728128, + "inactive_split_bytes_peak_delta": 166880768, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123029422080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123029422080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2909, + "segment_count_current_delta": 0, + "segment_count_peak": 2909, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190008909179Z.pickle", + "timestamp_utc": "2026-08-26T19:00:09.088419Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190008909179Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190008909179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d517a708764e416e5ec9b5cdfcf7ce348c7483d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190008909179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d30cdedc9a7047d6466972409d838b2fbdf618d27df996d57550b619cba6085c +size 9833199 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190001924184Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190001924184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f73594bfcf62293820de545316d6eda2fe69edea --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190001924184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84724621824, + "active_bytes_current_delta": 41171276800, + "active_bytes_peak": 90918017536, + "active_bytes_peak_delta": 47358381056, + "after": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 90918017536, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 90918017536, + "device": 1, + "inactive_split_bytes_current": 4263828992, + "inactive_split_bytes_peak": 14250550272, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 92207579136, + "reserved_bytes_peak": 92207579136, + "segment_count_current": 2181, + "segment_count_peak": 2181, + "timestamp_utc": "2026-08-26T19:00:01.923669Z", + "world_size": 8 + }, + "allocated_bytes_current": 84724621824, + "allocated_bytes_current_delta": 41171276800, + "allocated_bytes_peak": 90918017536, + "allocated_bytes_peak_delta": 47358381056, + "before": { + "active_bytes_current": 43553345024, + "active_bytes_peak": 43559636480, + "allocated_bytes_current": 43553345024, + "allocated_bytes_peak": 43559636480, + "device": 1, + "inactive_split_bytes_current": 1254404608, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:59:17.789289Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4263828992, + "inactive_split_bytes_current_delta": 3009424384, + "inactive_split_bytes_peak": 14250550272, + "inactive_split_bytes_peak_delta": 12637637632, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 92207579136, + "reserved_bytes_current_delta": 47383052288, + "reserved_bytes_peak": 92207579136, + "reserved_bytes_peak_delta": 47383052288, + "segment_count_current": 2181, + "segment_count_current_delta": 1291, + "segment_count_peak": 2181, + "segment_count_peak_delta": 1291, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190001924184Z.pickle", + "timestamp_utc": "2026-08-26T19:00:02.110346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190001924184Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190001924184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb6d302e4259ffb6bd68e61846e7c3a3f844dc6e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190001924184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ca3aa1c146a2cc771d687ed489a9faec11887e401ff7e3727a59c0d2d7daf4 +size 9819755 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190004404042Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190004404042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53b3b2aeec6d9c91a4533e258358e8a3718801da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190004404042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84724621824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121713718784, + "active_bytes_peak_delta": 30795701248, + "after": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 121713718784, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 121713718784, + "device": 1, + "inactive_split_bytes_current": 4165262848, + "inactive_split_bytes_peak": 14250550272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122985381888, + "reserved_bytes_peak": 122985381888, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T19:00:04.403602Z", + "world_size": 8 + }, + "allocated_bytes_current": 84724621824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121713718784, + "allocated_bytes_peak_delta": 30795701248, + "before": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 90918017536, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 90918017536, + "device": 1, + "inactive_split_bytes_current": 4263828992, + "inactive_split_bytes_peak": 14250550272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 92207579136, + "reserved_bytes_peak": 92207579136, + "segment_count_current": 2181, + "segment_count_peak": 2181, + "timestamp_utc": "2026-08-26T19:00:02.147538Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4165262848, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14250550272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122985381888, + "reserved_bytes_current_delta": 30777802752, + "reserved_bytes_peak": 122985381888, + "reserved_bytes_peak_delta": 30777802752, + "segment_count_current": 2890, + "segment_count_current_delta": 709, + "segment_count_peak": 2890, + "segment_count_peak_delta": 709, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190004404042Z.pickle", + "timestamp_utc": "2026-08-26T19:00:04.602129Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190004404042Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190004404042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3a8db02332368b8329af77b05c26faa2eb828c7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190004404042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:767e34186440e532e759f374f373629327623617aa19f3d6839389b9f1410e77 +size 9861264 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190005872700Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190005872700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e26831aa4946bbaecfaae0f3a2e0065a1ef5fced --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190005872700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84724621824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121713718784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 121713718784, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 121713718784, + "device": 1, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_peak": 14250550272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122985381888, + "reserved_bytes_peak": 122985381888, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T19:00:05.872242Z", + "world_size": 8 + }, + "allocated_bytes_current": 84724621824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121713718784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 121713718784, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 121713718784, + "device": 1, + "inactive_split_bytes_current": 4165262848, + "inactive_split_bytes_peak": 14250550272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122985381888, + "reserved_bytes_peak": 122985381888, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T19:00:04.629557Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14250550272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122985381888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122985381888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2890, + "segment_count_current_delta": 0, + "segment_count_peak": 2890, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190005872700Z.pickle", + "timestamp_utc": "2026-08-26T19:00:06.067454Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190005872700Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190005872700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e3fc9282295a521d3d44558cb367b506a1bea10 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190005872700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c10106bc278936b8d2b533e563e505e2b3a78de7c93426cd1d165e67facf6317 +size 9835356 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190007540634Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190007540634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1831319426aa1a5ca0b775dfe78fedf94e428434 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190007540634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84724621824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121713718784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 121713718784, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 121713718784, + "device": 1, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_peak": 14908156928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122985381888, + "reserved_bytes_peak": 122985381888, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T19:00:07.540182Z", + "world_size": 8 + }, + "allocated_bytes_current": 84724621824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121713718784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 121713718784, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 121713718784, + "device": 1, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_peak": 14250550272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122985381888, + "reserved_bytes_peak": 122985381888, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T19:00:06.094595Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14908156928, + "inactive_split_bytes_peak_delta": 657606656, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122985381888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122985381888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2890, + "segment_count_current_delta": 0, + "segment_count_peak": 2890, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190007540634Z.pickle", + "timestamp_utc": "2026-08-26T19:00:07.732318Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190007540634Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190007540634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f340df3bbd98b163eea93e393ac632fb0d26bc6b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190007540634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c215bc0b3a68ea3d4f58d895661d706b723ca1e6b0530fff7f2d32ce1f5fb121 +size 9834423 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190008909187Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190008909187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03499fbbe629ac1330eb3883918e10d9ded35a58 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190008909187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84724621824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121713718784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 121713718784, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 121713718784, + "device": 1, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_peak": 15237392384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122985381888, + "reserved_bytes_peak": 122985381888, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T19:00:08.908740Z", + "world_size": 8 + }, + "allocated_bytes_current": 84724621824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121713718784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84724621824, + "active_bytes_peak": 121713718784, + "allocated_bytes_current": 84724621824, + "allocated_bytes_peak": 121713718784, + "device": 1, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_peak": 14908156928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122985381888, + "reserved_bytes_peak": 122985381888, + "segment_count_current": 2890, + "segment_count_peak": 2890, + "timestamp_utc": "2026-08-26T19:00:07.758627Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4265926144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15237392384, + "inactive_split_bytes_peak_delta": 329235456, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122985381888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122985381888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2890, + "segment_count_current_delta": 0, + "segment_count_peak": 2890, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190008909187Z.pickle", + "timestamp_utc": "2026-08-26T19:00:09.098865Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190008909187Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190008909187Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bca3e3e5f60608b805e1c62f4d55abcad0887f49 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190008909187Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:226384835e9d4323387e87534a3b04f2aa64b726148f62a20ec842ba67763a5a +size 9833855 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190001924151Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190001924151Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a1fd02a587baf0eca615a3b9db7ce8b0af83313 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190001924151Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715355136, + "active_bytes_current_delta": 41161811968, + "active_bytes_peak": 90909895680, + "active_bytes_peak_delta": 47350061056, + "after": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 90909895680, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 90909895680, + "device": 2, + "inactive_split_bytes_current": 4547822592, + "inactive_split_bytes_peak": 14537034240, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 92165636096, + "reserved_bytes_peak": 92165636096, + "segment_count_current": 2164, + "segment_count_peak": 2164, + "timestamp_utc": "2026-08-26T19:00:01.923615Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715355136, + "allocated_bytes_current_delta": 41161811968, + "allocated_bytes_peak": 90909895680, + "allocated_bytes_peak_delta": 47350061056, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 2, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:59:17.789187Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4547822592, + "inactive_split_bytes_current_delta": 3291518976, + "inactive_split_bytes_peak": 14537034240, + "inactive_split_bytes_peak_delta": 12924121600, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 92165636096, + "reserved_bytes_current_delta": 47330623488, + "reserved_bytes_peak": 92165636096, + "reserved_bytes_peak_delta": 47330623488, + "segment_count_current": 2164, + "segment_count_current_delta": 1281, + "segment_count_peak": 2164, + "segment_count_peak_delta": 1281, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190001924151Z.pickle", + "timestamp_utc": "2026-08-26T19:00:02.107097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190001924151Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190001924151Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5988113395674bfc2a95cae13dda9ce076eb751d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190001924151Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6137d6a8cbfdd9eba7f6cbba78c3142e007429df821549a12bc78329da71fe39 +size 9359325 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190004404134Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190004404134Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0535b99263ede08a504f281332fc923ad8fc4439 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190004404134Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715355136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121737633280, + "active_bytes_peak_delta": 30827737600, + "after": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 121737633280, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 121737633280, + "device": 2, + "inactive_split_bytes_current": 4449256448, + "inactive_split_bytes_peak": 14537034240, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:00:04.403654Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715355136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121737633280, + "allocated_bytes_peak_delta": 30827737600, + "before": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 90909895680, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 90909895680, + "device": 2, + "inactive_split_bytes_current": 4547822592, + "inactive_split_bytes_peak": 14537034240, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 92165636096, + "reserved_bytes_peak": 92165636096, + "segment_count_current": 2164, + "segment_count_peak": 2164, + "timestamp_utc": "2026-08-26T19:00:02.145154Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4449256448, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14537034240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 30855397376, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 30855397376, + "segment_count_current": 2887, + "segment_count_current_delta": 723, + "segment_count_peak": 2887, + "segment_count_peak_delta": 723, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190004404134Z.pickle", + "timestamp_utc": "2026-08-26T19:00:04.607166Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190004404134Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190004404134Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f11399f8ee4c0dace613d688bc931f9bfefccbf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190004404134Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38236224eb5efce49b04765e9a48149313f3068a832db1e6a74e740b6f053c7 +size 9850015 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190005872840Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190005872840Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b73580400a884294c7382bb2f41e2d22ef6b4fa8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190005872840Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715355136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121737633280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 121737633280, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 121737633280, + "device": 2, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_peak": 14537034240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:00:05.872384Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715355136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121737633280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 121737633280, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 121737633280, + "device": 2, + "inactive_split_bytes_current": 4449256448, + "inactive_split_bytes_peak": 14537034240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:00:04.634689Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14537034240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2887, + "segment_count_current_delta": 0, + "segment_count_peak": 2887, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190005872840Z.pickle", + "timestamp_utc": "2026-08-26T19:00:06.075104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190005872840Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190005872840Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cfec489e8a7bf17e8db527a8fb139be034d3335 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190005872840Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:255304b16d2acf70f4459eded6baf590f316a40ef130731cc285653e20b4bedc +size 9823041 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190007540777Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190007540777Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27ec55db6bcb6a6b6d02b385872317f5bb5752fc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190007540777Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715355136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121737633280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 121737633280, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 121737633280, + "device": 2, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_peak": 15262635008, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:00:07.540321Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715355136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121737633280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 121737633280, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 121737633280, + "device": 2, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_peak": 14537034240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:00:06.101948Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15262635008, + "inactive_split_bytes_peak_delta": 725600768, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2887, + "segment_count_current_delta": 0, + "segment_count_peak": 2887, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190007540777Z.pickle", + "timestamp_utc": "2026-08-26T19:00:07.745223Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190007540777Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190007540777Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cfd7d2e41a58702f9aab9c9828b8e427f56bf4a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190007540777Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b8a619cb79684e38de1e96ae284753201412e31a822808ad45e4d5cb8707a4a +size 9822337 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190008909367Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190008909367Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5d62f669e3cf637cd84d3dce0e29572396fe3a4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190008909367Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715355136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121737633280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 121737633280, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 121737633280, + "device": 2, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_peak": 15262635008, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:00:08.908880Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715355136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121737633280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84715355136, + "active_bytes_peak": 121737633280, + "allocated_bytes_current": 84715355136, + "allocated_bytes_peak": 121737633280, + "device": 2, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_peak": 15262635008, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:00:07.771836Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4549919744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15262635008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2887, + "segment_count_current_delta": 0, + "segment_count_peak": 2887, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190008909367Z.pickle", + "timestamp_utc": "2026-08-26T19:00:09.111857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190008909367Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190008909367Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31759f8f55fb94b081db574e22e12076fa4fbdd7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190008909367Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:405ac3c4981ac27530f1f885c68ce0f3d9bf6aa8fb9563bf85ea67f8e2e49f4e +size 9821936 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190001924127Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190001924127Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a9f67c63b55a2ffe8df021579c37aeff9eb7f13 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190001924127Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84728074752, + "active_bytes_current_delta": 41171252224, + "active_bytes_peak": 90919874048, + "active_bytes_peak_delta": 47356760064, + "after": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 90919874048, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 90919874048, + "device": 3, + "inactive_split_bytes_current": 4239404544, + "inactive_split_bytes_peak": 14262669312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 91983183872, + "reserved_bytes_peak": 91983183872, + "segment_count_current": 2179, + "segment_count_peak": 2179, + "timestamp_utc": "2026-08-26T19:00:01.923606Z", + "world_size": 8 + }, + "allocated_bytes_current": 84728074752, + "allocated_bytes_current_delta": 41171252224, + "allocated_bytes_peak": 90919874048, + "allocated_bytes_peak_delta": 47356760064, + "before": { + "active_bytes_current": 43556822528, + "active_bytes_peak": 43563113984, + "allocated_bytes_current": 43556822528, + "allocated_bytes_peak": 43563113984, + "device": 3, + "inactive_split_bytes_current": 1223664128, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T18:59:17.789180Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4239404544, + "inactive_split_bytes_current_delta": 3015740416, + "inactive_split_bytes_peak": 14262669312, + "inactive_split_bytes_peak_delta": 12649756672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 91983183872, + "reserved_bytes_current_delta": 47188017152, + "reserved_bytes_peak": 91983183872, + "reserved_bytes_peak_delta": 47188017152, + "segment_count_current": 2179, + "segment_count_current_delta": 1292, + "segment_count_peak": 2179, + "segment_count_peak_delta": 1292, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190001924127Z.pickle", + "timestamp_utc": "2026-08-26T19:00:02.113229Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190001924127Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190001924127Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28b266fb3727a08d006f9f820b8cb482a11f52fe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190001924127Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f05089fbae90a5b3085eac722f0bdeab987ed79ad7f77e9b1a72ad1abe5001d9 +size 9376908 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190004404064Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190004404064Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6dddfdc14d08d011f673e84600c0b708acb34c5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190004404064Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84728074752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121697109504, + "active_bytes_peak_delta": 30777235456, + "after": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 121697109504, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 121697109504, + "device": 3, + "inactive_split_bytes_current": 4140838400, + "inactive_split_bytes_peak": 14262669312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122811318272, + "reserved_bytes_peak": 122811318272, + "segment_count_current": 2897, + "segment_count_peak": 2897, + "timestamp_utc": "2026-08-26T19:00:04.403627Z", + "world_size": 8 + }, + "allocated_bytes_current": 84728074752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121697109504, + "allocated_bytes_peak_delta": 30777235456, + "before": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 90919874048, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 90919874048, + "device": 3, + "inactive_split_bytes_current": 4239404544, + "inactive_split_bytes_peak": 14262669312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 91983183872, + "reserved_bytes_peak": 91983183872, + "segment_count_current": 2179, + "segment_count_peak": 2179, + "timestamp_utc": "2026-08-26T19:00:02.149763Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4140838400, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14262669312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122811318272, + "reserved_bytes_current_delta": 30828134400, + "reserved_bytes_peak": 122811318272, + "reserved_bytes_peak_delta": 30828134400, + "segment_count_current": 2897, + "segment_count_current_delta": 718, + "segment_count_peak": 2897, + "segment_count_peak_delta": 718, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190004404064Z.pickle", + "timestamp_utc": "2026-08-26T19:00:04.602233Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190004404064Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190004404064Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc2173a7ac5ded0aa19ef6308ee793f86aca1e94 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190004404064Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45279315865395551b57af31b7c263f939a180b865599814454712c03caaa0b0 +size 9862447 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190005872695Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190005872695Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35363e58422d4f2c0f264af570142e41451d1e9b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190005872695Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84728074752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121697109504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 121697109504, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 121697109504, + "device": 3, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_peak": 14750922240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122811318272, + "reserved_bytes_peak": 122811318272, + "segment_count_current": 2897, + "segment_count_peak": 2897, + "timestamp_utc": "2026-08-26T19:00:05.872237Z", + "world_size": 8 + }, + "allocated_bytes_current": 84728074752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121697109504, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 121697109504, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 121697109504, + "device": 3, + "inactive_split_bytes_current": 4140838400, + "inactive_split_bytes_peak": 14262669312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122811318272, + "reserved_bytes_peak": 122811318272, + "segment_count_current": 2897, + "segment_count_peak": 2897, + "timestamp_utc": "2026-08-26T19:00:04.629788Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14750922240, + "inactive_split_bytes_peak_delta": 488252928, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122811318272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122811318272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2897, + "segment_count_current_delta": 0, + "segment_count_peak": 2897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190005872695Z.pickle", + "timestamp_utc": "2026-08-26T19:00:06.069703Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190005872695Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190005872695Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60585bfe923bc6d830d3d671058e4b032301f890 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190005872695Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0eb7d9431a78c3833c0ebf463276c94d6cae6087cc6f4b3516e166e4da6f212 +size 9836147 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190007540702Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190007540702Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa84afeac8e217f81b68c13a0026d3629501d1fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190007540702Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84728074752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121697109504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 121697109504, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 121697109504, + "device": 3, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_peak": 15504766976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122811318272, + "reserved_bytes_peak": 122811318272, + "segment_count_current": 2897, + "segment_count_peak": 2897, + "timestamp_utc": "2026-08-26T19:00:07.540234Z", + "world_size": 8 + }, + "allocated_bytes_current": 84728074752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121697109504, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 121697109504, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 121697109504, + "device": 3, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_peak": 14750922240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122811318272, + "reserved_bytes_peak": 122811318272, + "segment_count_current": 2897, + "segment_count_peak": 2897, + "timestamp_utc": "2026-08-26T19:00:06.096329Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15504766976, + "inactive_split_bytes_peak_delta": 753844736, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122811318272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122811318272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2897, + "segment_count_current_delta": 0, + "segment_count_peak": 2897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190007540702Z.pickle", + "timestamp_utc": "2026-08-26T19:00:07.733349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190007540702Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190007540702Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..92d41101aa74c5291aedf7cdee5df9856828c1ac --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190007540702Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a54ac05c1facdb7f18206da3695f68ba0edce17e18a2e84e35b9f2911f2949 +size 9835076 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190008909242Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190008909242Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4db04b1e4a880af2f5bc4703148d188fedbfdc1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190008909242Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84728074752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121697109504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 121697109504, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 121697109504, + "device": 3, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_peak": 15506649600, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122811318272, + "reserved_bytes_peak": 122811318272, + "segment_count_current": 2897, + "segment_count_peak": 2897, + "timestamp_utc": "2026-08-26T19:00:08.908770Z", + "world_size": 8 + }, + "allocated_bytes_current": 84728074752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121697109504, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84728074752, + "active_bytes_peak": 121697109504, + "allocated_bytes_current": 84728074752, + "allocated_bytes_peak": 121697109504, + "device": 3, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_peak": 15504766976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122811318272, + "reserved_bytes_peak": 122811318272, + "segment_count_current": 2897, + "segment_count_peak": 2897, + "timestamp_utc": "2026-08-26T19:00:07.759851Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4241501696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15506649600, + "inactive_split_bytes_peak_delta": 1882624, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122811318272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122811318272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2897, + "segment_count_current_delta": 0, + "segment_count_peak": 2897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190008909242Z.pickle", + "timestamp_utc": "2026-08-26T19:00:09.101911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190008909242Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190008909242Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80537420c8a7be207e5a1a30fade1adc390e8ca7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190008909242Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51739a22e23f50cb49daee5b85c6f7454d2fb5e608d8fb26226ed60bf95ab647 +size 9834508 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190001924144Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190001924144Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9533640244ca468156501455e94d67eefeda5160 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190001924144Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715471360, + "active_bytes_current_delta": 41164793856, + "active_bytes_peak": 90904766976, + "active_bytes_peak_delta": 47347798016, + "after": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 90904766976, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 90904766976, + "device": 4, + "inactive_split_bytes_current": 4264590848, + "inactive_split_bytes_peak": 14003799040, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 92201287680, + "reserved_bytes_peak": 92201287680, + "segment_count_current": 2180, + "segment_count_peak": 2180, + "timestamp_utc": "2026-08-26T19:00:01.923649Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715471360, + "allocated_bytes_current_delta": 41164793856, + "allocated_bytes_peak": 90904766976, + "allocated_bytes_peak_delta": 47347798016, + "before": { + "active_bytes_current": 43550677504, + "active_bytes_peak": 43556968960, + "allocated_bytes_current": 43550677504, + "allocated_bytes_peak": 43556968960, + "device": 4, + "inactive_split_bytes_current": 1252877824, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T18:59:17.789210Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4264590848, + "inactive_split_bytes_current_delta": 3011713024, + "inactive_split_bytes_peak": 14003799040, + "inactive_split_bytes_peak_delta": 12390886400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 92201287680, + "reserved_bytes_current_delta": 47380955136, + "reserved_bytes_peak": 92201287680, + "reserved_bytes_peak_delta": 47380955136, + "segment_count_current": 2180, + "segment_count_current_delta": 1291, + "segment_count_peak": 2180, + "segment_count_peak_delta": 1291, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190001924144Z.pickle", + "timestamp_utc": "2026-08-26T19:00:02.107445Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190001924144Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190001924144Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..307866111cbdc50f7f2b8f4440236c391334db72 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190001924144Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f002bcef65b0b5184e3ebf1524c104117e07d0976efec732c91c3deeedd138c +size 10878064 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190004404102Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190004404102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0bda719c817604fee111e2eb6cd94be54c3d3c3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190004404102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715471360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121721346560, + "active_bytes_peak_delta": 30816579584, + "after": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 121721346560, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 121721346560, + "device": 4, + "inactive_split_bytes_current": 4166024704, + "inactive_split_bytes_peak": 14003799040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2903, + "segment_count_peak": 2903, + "timestamp_utc": "2026-08-26T19:00:04.403653Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715471360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121721346560, + "allocated_bytes_peak_delta": 30816579584, + "before": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 90904766976, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 90904766976, + "device": 4, + "inactive_split_bytes_current": 4264590848, + "inactive_split_bytes_peak": 14003799040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 92201287680, + "reserved_bytes_peak": 92201287680, + "segment_count_current": 2180, + "segment_count_peak": 2180, + "timestamp_utc": "2026-08-26T19:00:02.140986Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4166024704, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14003799040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 30819745792, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 30819745792, + "segment_count_current": 2903, + "segment_count_current_delta": 723, + "segment_count_peak": 2903, + "segment_count_peak_delta": 723, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190004404102Z.pickle", + "timestamp_utc": "2026-08-26T19:00:04.593274Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190004404102Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190004404102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f834baa3f23ec818c612de14a9086db2ab878e0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190004404102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7a023b39f0a2c54690a11484d7046b9c8ba4fd2b2e7ce58ad5572b8aefa1d03 +size 9863265 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190005872708Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190005872708Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca956a89dc33a4b6f93660585d403f802047969e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190005872708Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715471360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121721346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 121721346560, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 121721346560, + "device": 4, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_peak": 14434465792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2903, + "segment_count_peak": 2903, + "timestamp_utc": "2026-08-26T19:00:05.872264Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715471360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121721346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 121721346560, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 121721346560, + "device": 4, + "inactive_split_bytes_current": 4166024704, + "inactive_split_bytes_peak": 14003799040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2903, + "segment_count_peak": 2903, + "timestamp_utc": "2026-08-26T19:00:04.617840Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14434465792, + "inactive_split_bytes_peak_delta": 430666752, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2903, + "segment_count_current_delta": 0, + "segment_count_peak": 2903, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190005872708Z.pickle", + "timestamp_utc": "2026-08-26T19:00:06.063214Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190005872708Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190005872708Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2c7972da2cfe21572107b03195a431465061d28 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190005872708Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d1099b0ab6fbca460fadebb1f943092e334ae9d73fe3ae67ce21763729ccd3b +size 9836934 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190007540646Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190007540646Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7a4a0ec6e55c913f1385ebd63893356b919ef045 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190007540646Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715471360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121721346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 121721346560, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 121721346560, + "device": 4, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_peak": 15235512320, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2903, + "segment_count_peak": 2903, + "timestamp_utc": "2026-08-26T19:00:07.540211Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715471360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121721346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 121721346560, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 121721346560, + "device": 4, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_peak": 14434465792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2903, + "segment_count_peak": 2903, + "timestamp_utc": "2026-08-26T19:00:06.089693Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15235512320, + "inactive_split_bytes_peak_delta": 801046528, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2903, + "segment_count_current_delta": 0, + "segment_count_peak": 2903, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190007540646Z.pickle", + "timestamp_utc": "2026-08-26T19:00:07.728342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190007540646Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190007540646Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cd544f9d0b8dccd2c9b3a5d0c699672482e398e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190007540646Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a839aa3b5bdcbc8c38b07439355dcd15cc72649575fec9c1fffe40079352cb +size 9835475 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190008909193Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190008909193Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8112565f26227c90cdb09da5b7ed29d07949d64 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190008909193Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84715471360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121721346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 121721346560, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 121721346560, + "device": 4, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_peak": 15373919232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2903, + "segment_count_peak": 2903, + "timestamp_utc": "2026-08-26T19:00:08.908750Z", + "world_size": 8 + }, + "allocated_bytes_current": 84715471360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121721346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84715471360, + "active_bytes_peak": 121721346560, + "allocated_bytes_current": 84715471360, + "allocated_bytes_peak": 121721346560, + "device": 4, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_peak": 15235512320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 123021033472, + "reserved_bytes_peak": 123021033472, + "segment_count_current": 2903, + "segment_count_peak": 2903, + "timestamp_utc": "2026-08-26T19:00:07.753793Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4266688000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15373919232, + "inactive_split_bytes_peak_delta": 138406912, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 123021033472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123021033472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2903, + "segment_count_current_delta": 0, + "segment_count_peak": 2903, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190008909193Z.pickle", + "timestamp_utc": "2026-08-26T19:00:09.094028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190008909193Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190008909193Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84c3cdaea22589ba9a8b487d1d26a8ac4cc5342f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190008909193Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7048038cd67a7b34396ac710c2e8e119a7df1fb7db5f088fa6e275a1c19fe72e +size 9835044 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190001924140Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190001924140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a10f13ceb83a8b38f1ceaf04c037f8ca72594d27 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190001924140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84713718784, + "active_bytes_current_delta": 41160175616, + "active_bytes_peak": 90917094912, + "active_bytes_peak_delta": 47357260288, + "after": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 90917094912, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 90917094912, + "device": 5, + "inactive_split_bytes_current": 4421532672, + "inactive_split_bytes_peak": 14193912320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 92180316160, + "reserved_bytes_peak": 92180316160, + "segment_count_current": 2168, + "segment_count_peak": 2168, + "timestamp_utc": "2026-08-26T19:00:01.923598Z", + "world_size": 8 + }, + "allocated_bytes_current": 84713718784, + "allocated_bytes_current_delta": 41160175616, + "allocated_bytes_peak": 90917094912, + "allocated_bytes_peak_delta": 47357260288, + "before": { + "active_bytes_current": 43553543168, + "active_bytes_peak": 43559834624, + "allocated_bytes_current": 43553543168, + "allocated_bytes_peak": 43559834624, + "device": 5, + "inactive_split_bytes_current": 1256303616, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T18:59:17.789257Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4421532672, + "inactive_split_bytes_current_delta": 3165229056, + "inactive_split_bytes_peak": 14193912320, + "inactive_split_bytes_peak_delta": 12580999680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 92180316160, + "reserved_bytes_current_delta": 47345303552, + "reserved_bytes_peak": 92180316160, + "reserved_bytes_peak_delta": 47345303552, + "segment_count_current": 2168, + "segment_count_current_delta": 1285, + "segment_count_peak": 2168, + "segment_count_peak_delta": 1285, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190001924140Z.pickle", + "timestamp_utc": "2026-08-26T19:00:02.109775Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190001924140Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190001924140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d521b16aa3a1c5acdfc3e237323ef22928e27e65 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190001924140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:282bc17c4b5e551d91d23a5712d84a60fdbc3779fb00746a7ad05f0c6eab43df +size 9410035 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190004404046Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190004404046Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad4237797e041aefa7547515849054269d3ec59d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190004404046Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84713718784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733142016, + "active_bytes_peak_delta": 30816047104, + "after": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 121733142016, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 121733142016, + "device": 5, + "inactive_split_bytes_current": 4322966528, + "inactive_split_bytes_peak": 14193912320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 123002159104, + "reserved_bytes_peak": 123002159104, + "segment_count_current": 2877, + "segment_count_peak": 2877, + "timestamp_utc": "2026-08-26T19:00:04.403590Z", + "world_size": 8 + }, + "allocated_bytes_current": 84713718784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733142016, + "allocated_bytes_peak_delta": 30816047104, + "before": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 90917094912, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 90917094912, + "device": 5, + "inactive_split_bytes_current": 4421532672, + "inactive_split_bytes_peak": 14193912320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 92180316160, + "reserved_bytes_peak": 92180316160, + "segment_count_current": 2168, + "segment_count_peak": 2168, + "timestamp_utc": "2026-08-26T19:00:02.148570Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4322966528, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14193912320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 123002159104, + "reserved_bytes_current_delta": 30821842944, + "reserved_bytes_peak": 123002159104, + "reserved_bytes_peak_delta": 30821842944, + "segment_count_current": 2877, + "segment_count_current_delta": 709, + "segment_count_peak": 2877, + "segment_count_peak_delta": 709, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190004404046Z.pickle", + "timestamp_utc": "2026-08-26T19:00:04.603044Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190004404046Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190004404046Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de758d9d9ce445a7902e0d1c278272cf90eaa973 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190004404046Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2073b3f00d7458936d8385dc7c206fe7f02c70fbe96239e0bb085c3cf8cd2f52 +size 9847817 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190005872696Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190005872696Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e09282ebf534e865924e1bf3da866bb25b02148 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190005872696Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84713718784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733142016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 121733142016, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 121733142016, + "device": 5, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_peak": 14447692800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 123002159104, + "reserved_bytes_peak": 123002159104, + "segment_count_current": 2877, + "segment_count_peak": 2877, + "timestamp_utc": "2026-08-26T19:00:05.872228Z", + "world_size": 8 + }, + "allocated_bytes_current": 84713718784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733142016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 121733142016, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 121733142016, + "device": 5, + "inactive_split_bytes_current": 4322966528, + "inactive_split_bytes_peak": 14193912320, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 123002159104, + "reserved_bytes_peak": 123002159104, + "segment_count_current": 2877, + "segment_count_peak": 2877, + "timestamp_utc": "2026-08-26T19:00:04.630619Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14447692800, + "inactive_split_bytes_peak_delta": 253780480, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 123002159104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123002159104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2877, + "segment_count_current_delta": 0, + "segment_count_peak": 2877, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190005872696Z.pickle", + "timestamp_utc": "2026-08-26T19:00:06.067336Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190005872696Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190005872696Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a23807e1703382c2109607145dcd61c2aaa1d98 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190005872696Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60baa4d8000e037f216af79c9d756521109f2debb920ddc2d089ddb0e5a5422e +size 9822066 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190007540644Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190007540644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae6537d2c9108a9b5295c40ff0883e78db425ff8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190007540644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84713718784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733142016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 121733142016, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 121733142016, + "device": 5, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_peak": 15129869312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 123002159104, + "reserved_bytes_peak": 123002159104, + "segment_count_current": 2877, + "segment_count_peak": 2877, + "timestamp_utc": "2026-08-26T19:00:07.540185Z", + "world_size": 8 + }, + "allocated_bytes_current": 84713718784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733142016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 121733142016, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 121733142016, + "device": 5, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_peak": 14447692800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 123002159104, + "reserved_bytes_peak": 123002159104, + "segment_count_current": 2877, + "segment_count_peak": 2877, + "timestamp_utc": "2026-08-26T19:00:06.094442Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15129869312, + "inactive_split_bytes_peak_delta": 682176512, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 123002159104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123002159104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2877, + "segment_count_current_delta": 0, + "segment_count_peak": 2877, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190007540644Z.pickle", + "timestamp_utc": "2026-08-26T19:00:07.731254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190007540644Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190007540644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ee05a9f65951af839a3bfc35f6a482f44ad23c0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190007540644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa9eaa7c0261fe03bf340b1e402ac30a7273b7394a1936c2c10b1e17df006d5b +size 9820961 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190008909190Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190008909190Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d957122b2a29434a1ebcab62aacd4c36a26cb1b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190008909190Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84713718784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121733142016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 121733142016, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 121733142016, + "device": 5, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_peak": 15157999616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 123002159104, + "reserved_bytes_peak": 123002159104, + "segment_count_current": 2877, + "segment_count_peak": 2877, + "timestamp_utc": "2026-08-26T19:00:08.908741Z", + "world_size": 8 + }, + "allocated_bytes_current": 84713718784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121733142016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84713718784, + "active_bytes_peak": 121733142016, + "allocated_bytes_current": 84713718784, + "allocated_bytes_peak": 121733142016, + "device": 5, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_peak": 15129869312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 123002159104, + "reserved_bytes_peak": 123002159104, + "segment_count_current": 2877, + "segment_count_peak": 2877, + "timestamp_utc": "2026-08-26T19:00:07.758032Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4423629824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15157999616, + "inactive_split_bytes_peak_delta": 28130304, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 123002159104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123002159104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2877, + "segment_count_current_delta": 0, + "segment_count_peak": 2877, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190008909190Z.pickle", + "timestamp_utc": "2026-08-26T19:00:09.096178Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190008909190Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190008909190Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89945560affc844aa16cf1d7b4b1272672378bd5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190008909190Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fe59dd504caf8b4f8905f946f8232f362a72f962af19e3eb46cc030ccfc65af +size 9820424 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190001924146Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190001924146Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b1a206734065352566c949076e32c49cb43189a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190001924146Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84722628096, + "active_bytes_current_delta": 41169217536, + "active_bytes_peak": 90894286848, + "active_bytes_peak_delta": 47334584832, + "after": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 90894286848, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 90894286848, + "device": 6, + "inactive_split_bytes_current": 4364388864, + "inactive_split_bytes_peak": 14430097920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 91964309504, + "reserved_bytes_peak": 91964309504, + "segment_count_current": 2179, + "segment_count_peak": 2179, + "timestamp_utc": "2026-08-26T19:00:01.923657Z", + "world_size": 8 + }, + "allocated_bytes_current": 84722628096, + "allocated_bytes_current_delta": 41169217536, + "allocated_bytes_peak": 90894286848, + "allocated_bytes_peak_delta": 47334584832, + "before": { + "active_bytes_current": 43553410560, + "active_bytes_peak": 43559702016, + "allocated_bytes_current": 43553410560, + "allocated_bytes_peak": 43559702016, + "device": 6, + "inactive_split_bytes_current": 1256436224, + "inactive_split_bytes_peak": 1612912640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T18:59:17.789100Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4364388864, + "inactive_split_bytes_current_delta": 3107952640, + "inactive_split_bytes_peak": 14430097920, + "inactive_split_bytes_peak_delta": 12817185280, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 91964309504, + "reserved_bytes_current_delta": 47139782656, + "reserved_bytes_peak": 91964309504, + "reserved_bytes_peak_delta": 47139782656, + "segment_count_current": 2179, + "segment_count_current_delta": 1289, + "segment_count_peak": 2179, + "segment_count_peak_delta": 1289, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190001924146Z.pickle", + "timestamp_utc": "2026-08-26T19:00:02.106891Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190001924146Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190001924146Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab21edaa4b781cc88818317de10a52e2f11064b4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190001924146Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d14e0bad786180b9052ec4dc1b66173b6570b37e2abbae00411e041d8288f459 +size 9377773 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190004404037Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190004404037Z.json new file mode 100644 index 0000000000000000000000000000000000000000..604f29086a8f0f26b1bf888df00437108d9b74c3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190004404037Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84722628096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121709740032, + "active_bytes_peak_delta": 30815453184, + "after": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 121709740032, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 121709740032, + "device": 6, + "inactive_split_bytes_current": 4265822720, + "inactive_split_bytes_peak": 14430097920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 122779860992, + "reserved_bytes_peak": 122779860992, + "segment_count_current": 2875, + "segment_count_peak": 2875, + "timestamp_utc": "2026-08-26T19:00:04.403601Z", + "world_size": 8 + }, + "allocated_bytes_current": 84722628096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121709740032, + "allocated_bytes_peak_delta": 30815453184, + "before": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 90894286848, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 90894286848, + "device": 6, + "inactive_split_bytes_current": 4364388864, + "inactive_split_bytes_peak": 14430097920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 91964309504, + "reserved_bytes_peak": 91964309504, + "segment_count_current": 2179, + "segment_count_peak": 2179, + "timestamp_utc": "2026-08-26T19:00:02.140754Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4265822720, + "inactive_split_bytes_current_delta": -98566144, + "inactive_split_bytes_peak": 14430097920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 122779860992, + "reserved_bytes_current_delta": 30815551488, + "reserved_bytes_peak": 122779860992, + "reserved_bytes_peak_delta": 30815551488, + "segment_count_current": 2875, + "segment_count_current_delta": 696, + "segment_count_peak": 2875, + "segment_count_peak_delta": 696, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190004404037Z.pickle", + "timestamp_utc": "2026-08-26T19:00:04.594300Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190004404037Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190004404037Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c3db711a06174e8311e7647092e600380b32c9f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190004404037Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82596fd8f290e2e09cd5be5074cc3104b63d7d8780e1692393109e5a11688974 +size 9859946 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190005872721Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190005872721Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fb3253d220c97ce0ebee2fd2c3ab5dfedf4c662 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190005872721Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84722628096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121709740032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 121709740032, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 121709740032, + "device": 6, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_peak": 14430097920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 122779860992, + "reserved_bytes_peak": 122779860992, + "segment_count_current": 2875, + "segment_count_peak": 2875, + "timestamp_utc": "2026-08-26T19:00:05.872273Z", + "world_size": 8 + }, + "allocated_bytes_current": 84722628096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121709740032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 121709740032, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 121709740032, + "device": 6, + "inactive_split_bytes_current": 4265822720, + "inactive_split_bytes_peak": 14430097920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 122779860992, + "reserved_bytes_peak": 122779860992, + "segment_count_current": 2875, + "segment_count_peak": 2875, + "timestamp_utc": "2026-08-26T19:00:04.618806Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 14430097920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 122779860992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122779860992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2875, + "segment_count_current_delta": 0, + "segment_count_peak": 2875, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190005872721Z.pickle", + "timestamp_utc": "2026-08-26T19:00:06.062935Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190005872721Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190005872721Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7526cff82a13566976b7526ceb0422158b587e2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190005872721Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4296e8bb69e268a4dd81f3ba88d8b5fcc95bed2c89bb08054657e9033419215a +size 9835199 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190007540639Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190007540639Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f93eb875513e82e40e1aee5c805a487c33944a05 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190007540639Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84722628096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121709740032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 121709740032, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 121709740032, + "device": 6, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_peak": 14906345984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 122779860992, + "reserved_bytes_peak": 122779860992, + "segment_count_current": 2875, + "segment_count_peak": 2875, + "timestamp_utc": "2026-08-26T19:00:07.540188Z", + "world_size": 8 + }, + "allocated_bytes_current": 84722628096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121709740032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 121709740032, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 121709740032, + "device": 6, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_peak": 14430097920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 122779860992, + "reserved_bytes_peak": 122779860992, + "segment_count_current": 2875, + "segment_count_peak": 2875, + "timestamp_utc": "2026-08-26T19:00:06.089418Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14906345984, + "inactive_split_bytes_peak_delta": 476248064, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 122779860992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122779860992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2875, + "segment_count_current_delta": 0, + "segment_count_peak": 2875, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190007540639Z.pickle", + "timestamp_utc": "2026-08-26T19:00:07.727421Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190007540639Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190007540639Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a4abac6efd32a2cfb14540b1bc305e2e1c298b5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190007540639Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf8faa7c0429b3b5139a3913c1695282b74584a0c89fd5c0d8491d040a823db9 +size 9833996 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6798a8b9671b2d12bf410d2583d4b46e0787554 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 84722628096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121709740032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 121709740032, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 121709740032, + "device": 6, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_peak": 14906345984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 122779860992, + "reserved_bytes_peak": 122779860992, + "segment_count_current": 2875, + "segment_count_peak": 2875, + "timestamp_utc": "2026-08-26T19:00:08.908739Z", + "world_size": 8 + }, + "allocated_bytes_current": 84722628096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121709740032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 84722628096, + "active_bytes_peak": 121709740032, + "allocated_bytes_current": 84722628096, + "allocated_bytes_peak": 121709740032, + "device": 6, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_peak": 14906345984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 122779860992, + "reserved_bytes_peak": 122779860992, + "segment_count_current": 2875, + "segment_count_peak": 2875, + "timestamp_utc": "2026-08-26T19:00:07.753288Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4366486016, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14906345984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 122779860992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122779860992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2875, + "segment_count_current_delta": 0, + "segment_count_peak": 2875, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.pickle", + "timestamp_utc": "2026-08-26T19:00:09.095474Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file