diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ae3a482cb31928ed0637daa0e6e824bfa7ce731 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42478680576, + "active_bytes_current_delta": 19430297088, + "active_bytes_peak": 113227885568, + "active_bytes_peak_delta": 90166919168, + "after": { + "active_bytes_current": 42478680576, + "active_bytes_peak": 113227885568, + "allocated_bytes_current": 42478680576, + "allocated_bytes_peak": 113227885568, + "device": 4, + "inactive_split_bytes_current": 18456167936, + "inactive_split_bytes_peak": 30415799296, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117411151872, + "reserved_bytes_peak": 117411151872, + "segment_count_current": 920, + "segment_count_peak": 920, + "timestamp_utc": "2026-08-26T19:04:02.080066Z", + "world_size": 8 + }, + "allocated_bytes_current": 42478680576, + "allocated_bytes_current_delta": 19430297088, + "allocated_bytes_peak": 113227885568, + "allocated_bytes_peak_delta": 90166919168, + "before": { + "active_bytes_current": 23048383488, + "active_bytes_peak": 23060966400, + "allocated_bytes_current": 23048383488, + "allocated_bytes_peak": 23060966400, + "device": 4, + "inactive_split_bytes_current": 376804352, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:03:23.745553Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18456167936, + "inactive_split_bytes_current_delta": 18079363584, + "inactive_split_bytes_peak": 30415799296, + "inactive_split_bytes_peak_delta": 29532563968, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117411151872, + "reserved_bytes_current_delta": 91788148736, + "reserved_bytes_peak": 117411151872, + "reserved_bytes_peak_delta": 91788148736, + "segment_count_current": 920, + "segment_count_current_delta": 369, + "segment_count_peak": 920, + "segment_count_peak_delta": 369, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.162509Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a2242b62b05d988cc8584ddbb162be4c1b34027 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d83729cb37450461395f2b17a82bced8dc5205c1a1dfc7f01154f3a556b77380 +size 4114465 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5cb8a0593864a2f30bb03e28328a5d45056de47 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42478680576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127729292800, + "active_bytes_peak_delta": 14501407232, + "after": { + "active_bytes_current": 42478680576, + "active_bytes_peak": 127729292800, + "allocated_bytes_current": 42478680576, + "allocated_bytes_peak": 127452468736, + "device": 4, + "inactive_split_bytes_current": 18548442624, + "inactive_split_bytes_peak": 30684399104, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 129807417344, + "reserved_bytes_peak": 129807417344, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:04:02.816236Z", + "world_size": 8 + }, + "allocated_bytes_current": 42478680576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127452468736, + "allocated_bytes_peak_delta": 14224583168, + "before": { + "active_bytes_current": 42478680576, + "active_bytes_peak": 113227885568, + "allocated_bytes_current": 42478680576, + "allocated_bytes_peak": 113227885568, + "device": 4, + "inactive_split_bytes_current": 18456167936, + "inactive_split_bytes_peak": 30415799296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117411151872, + "reserved_bytes_peak": 117411151872, + "segment_count_current": 920, + "segment_count_peak": 920, + "timestamp_utc": "2026-08-26T19:04:02.173774Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18548442624, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 30684399104, + "inactive_split_bytes_peak_delta": 268599808, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 129807417344, + "reserved_bytes_current_delta": 12396265472, + "reserved_bytes_peak": 129807417344, + "reserved_bytes_peak_delta": 12396265472, + "segment_count_current": 931, + "segment_count_current_delta": 11, + "segment_count_peak": 931, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.891028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e71ce9c35fa11c9933e994d4231fc0397757d412 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebe4fbadd82e9e91e33d1c17ef2c8d74cfd0adc06e63be0dc3e564db86ac3b95 +size 4503524 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db9594c2830c2b9ddc4de520c17130e66c914beb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42478680576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127731652096, + "active_bytes_peak_delta": 2359296, + "after": { + "active_bytes_current": 42478680576, + "active_bytes_peak": 127731652096, + "allocated_bytes_current": 42478680576, + "allocated_bytes_peak": 127454828032, + "device": 4, + "inactive_split_bytes_current": 18458265088, + "inactive_split_bytes_peak": 30684399104, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 129807417344, + "reserved_bytes_peak": 129807417344, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:04:03.544793Z", + "world_size": 8 + }, + "allocated_bytes_current": 42478680576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127454828032, + "allocated_bytes_peak_delta": 2359296, + "before": { + "active_bytes_current": 42478680576, + "active_bytes_peak": 127729292800, + "allocated_bytes_current": 42478680576, + "allocated_bytes_peak": 127452468736, + "device": 4, + "inactive_split_bytes_current": 18548442624, + "inactive_split_bytes_peak": 30684399104, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 129807417344, + "reserved_bytes_peak": 129807417344, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:04:02.899770Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18458265088, + "inactive_split_bytes_current_delta": -90177536, + "inactive_split_bytes_peak": 30684399104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 129807417344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129807417344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 931, + "segment_count_current_delta": 0, + "segment_count_peak": 931, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.620923Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72ad980007533f351886dba9e2888a73913ebc0b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fac8cfeefd3e11c9761bb1030a4b904986da1d50ce422eb6127c83e2cd002336 +size 4520747 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2074c9ee19f6dc7b04e54cb55c0e383267a6e0e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496641536, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130557001728, + "active_bytes_peak_delta": 2825349632, + "after": { + "active_bytes_current": 42496641536, + "active_bytes_peak": 130557001728, + "allocated_bytes_current": 42496641536, + "allocated_bytes_peak": 127454828032, + "device": 4, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_peak": 46659659264, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 155212316672, + "reserved_bytes_peak": 155212316672, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:04:05.730047Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496641536, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127454828032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42478680576, + "active_bytes_peak": 127731652096, + "allocated_bytes_current": 42478680576, + "allocated_bytes_peak": 127454828032, + "device": 4, + "inactive_split_bytes_current": 18458265088, + "inactive_split_bytes_peak": 30684399104, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 129807417344, + "reserved_bytes_peak": 129807417344, + "segment_count_current": 931, + "segment_count_peak": 931, + "timestamp_utc": "2026-08-26T19:04:03.629839Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_current_delta": 2152591360, + "inactive_split_bytes_peak": 46659659264, + "inactive_split_bytes_peak_delta": 15975260160, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 155212316672, + "reserved_bytes_current_delta": 25404899328, + "reserved_bytes_peak": 155212316672, + "reserved_bytes_peak_delta": 25404899328, + "segment_count_current": 1130, + "segment_count_current_delta": 199, + "segment_count_peak": 1130, + "segment_count_peak_delta": 199, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.818896Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2cad5238a6bdcf68b7fb28d721d54123a71f0e7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ac130196532840e49750a16588a05035df2819076786300a6423ec1be2210e +size 4737184 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0459ece9fb6f4b29a43a4b9a7b8d5336e6808917 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496641536, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130557001728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42496641536, + "active_bytes_peak": 130557001728, + "allocated_bytes_current": 42496641536, + "allocated_bytes_peak": 127454828032, + "device": 4, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_peak": 46659659264, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 155212316672, + "reserved_bytes_peak": 155212316672, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:04:06.418698Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496641536, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127454828032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42496641536, + "active_bytes_peak": 130557001728, + "allocated_bytes_current": 42496641536, + "allocated_bytes_peak": 127454828032, + "device": 4, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_peak": 46659659264, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 155212316672, + "reserved_bytes_peak": 155212316672, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:04:05.828949Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46659659264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 155212316672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155212316672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1130, + "segment_count_current_delta": 0, + "segment_count_peak": 1130, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.429348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3ce90fe3827729e2870ff3b6059b766cce88bd5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18b40bf0480d107b9ef1624b9ce4794741d1aa07c358c1fc50687acd34ec779e +size 381106 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1292a04054b4c167e7c313d4d26e76384a60b473 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42496641536, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130557001728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42496641536, + "active_bytes_peak": 130557001728, + "allocated_bytes_current": 42496641536, + "allocated_bytes_peak": 127454828032, + "device": 4, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_peak": 46659659264, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 155212316672, + "reserved_bytes_peak": 155212316672, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:04:07.024122Z", + "world_size": 8 + }, + "allocated_bytes_current": 42496641536, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127454828032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42496641536, + "active_bytes_peak": 130557001728, + "allocated_bytes_current": 42496641536, + "allocated_bytes_peak": 127454828032, + "device": 4, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_peak": 46659659264, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 155212316672, + "reserved_bytes_peak": 155212316672, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:04:06.430441Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 20610856448, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46659659264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 155212316672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155212316672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1130, + "segment_count_current_delta": 0, + "segment_count_peak": 1130, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.034324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea032e0f202cef84b9c22e50a41db46fc8ecfadf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c81cf1f8019b801f8aa391c3519ce153b5a72c6ec7200125305317d1c965999 +size 381106 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6589e78464ef625ad6f291f860835fdc9f05afdf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42487518720, + "active_bytes_current_delta": 19434335744, + "active_bytes_peak": 113240445952, + "active_bytes_peak_delta": 90174680064, + "after": { + "active_bytes_current": 42487518720, + "active_bytes_peak": 113240445952, + "allocated_bytes_current": 42487518720, + "allocated_bytes_peak": 113240445952, + "device": 5, + "inactive_split_bytes_current": 18376026624, + "inactive_split_bytes_peak": 30425401344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117409054720, + "reserved_bytes_peak": 117409054720, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:04:02.080015Z", + "world_size": 8 + }, + "allocated_bytes_current": 42487518720, + "allocated_bytes_current_delta": 19434335744, + "allocated_bytes_peak": 113240445952, + "allocated_bytes_peak_delta": 90174680064, + "before": { + "active_bytes_current": 23053182976, + "active_bytes_peak": 23065765888, + "allocated_bytes_current": 23053182976, + "allocated_bytes_peak": 23065765888, + "device": 5, + "inactive_split_bytes_current": 462182400, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:03:23.745648Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18376026624, + "inactive_split_bytes_current_delta": 17913844224, + "inactive_split_bytes_peak": 30425401344, + "inactive_split_bytes_peak_delta": 29542166016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117409054720, + "reserved_bytes_current_delta": 91786051584, + "reserved_bytes_peak": 117409054720, + "reserved_bytes_peak_delta": 91786051584, + "segment_count_current": 919, + "segment_count_current_delta": 368, + "segment_count_peak": 919, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.166044Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..553ce8f8f963e43425df958c1492686420dd72fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56b903ccb299ef35670c726458cd25437f737f6bf7790813de00992262510475 +size 4134331 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4196484f6e975287ebdaf9a17d68386d1f1b6d19 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42487518720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127585480192, + "active_bytes_peak_delta": 14345034240, + "after": { + "active_bytes_current": 42487518720, + "active_bytes_peak": 127585480192, + "allocated_bytes_current": 42487518720, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 18468301312, + "inactive_split_bytes_peak": 30425401344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 131946512384, + "reserved_bytes_peak": 131946512384, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:04:02.816170Z", + "world_size": 8 + }, + "allocated_bytes_current": 42487518720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447068160, + "allocated_bytes_peak_delta": 14206622208, + "before": { + "active_bytes_current": 42487518720, + "active_bytes_peak": 113240445952, + "allocated_bytes_current": 42487518720, + "allocated_bytes_peak": 113240445952, + "device": 5, + "inactive_split_bytes_current": 18376026624, + "inactive_split_bytes_peak": 30425401344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117409054720, + "reserved_bytes_peak": 117409054720, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-26T19:04:02.178637Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18468301312, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 30425401344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 131946512384, + "reserved_bytes_current_delta": 14537457664, + "reserved_bytes_peak": 131946512384, + "reserved_bytes_peak_delta": 14537457664, + "segment_count_current": 928, + "segment_count_current_delta": 9, + "segment_count_peak": 928, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.892646Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cdb7a14e15ad003c980e8f3eaaa1b1f5b49dfa77 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96de714f4a3d9f18d4e49013712ee500a525c925beb064029d5c21ddadb9cbe1 +size 4529243 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bbed35cdea1a2dd2dabaf326f05bcee56d809f6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42487518720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127718985216, + "active_bytes_peak_delta": 133505024, + "after": { + "active_bytes_current": 42487518720, + "active_bytes_peak": 127718985216, + "allocated_bytes_current": 42487518720, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 18470398464, + "inactive_split_bytes_peak": 31822153216, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 131946512384, + "reserved_bytes_peak": 131946512384, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:04:03.544740Z", + "world_size": 8 + }, + "allocated_bytes_current": 42487518720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447068160, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42487518720, + "active_bytes_peak": 127585480192, + "allocated_bytes_current": 42487518720, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 18468301312, + "inactive_split_bytes_peak": 30425401344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 131946512384, + "reserved_bytes_peak": 131946512384, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:04:02.902746Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 18470398464, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 31822153216, + "inactive_split_bytes_peak_delta": 1396751872, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 131946512384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 131946512384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.619957Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae5880a933f716abe776885bff2205483e890a81 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8afa962f2fe520b956cebd932c0a11446c25f3b7e7950d89aa43ec0c43724a6 +size 4545972 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.json new file mode 100644 index 0000000000000000000000000000000000000000..853451ff30443a2ce09241cad10de79039691639 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505479680, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130565839872, + "active_bytes_peak_delta": 2846854656, + "after": { + "active_bytes_current": 42505479680, + "active_bytes_peak": 130565839872, + "allocated_bytes_current": 42505479680, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_peak": 46669695488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 155233288192, + "reserved_bytes_peak": 155233288192, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:04:05.729868Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505479680, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127447068160, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42487518720, + "active_bytes_peak": 127718985216, + "allocated_bytes_current": 42487518720, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 18470398464, + "inactive_split_bytes_peak": 31822153216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 131946512384, + "reserved_bytes_peak": 131946512384, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:04:03.629418Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 46669695488, + "inactive_split_bytes_peak_delta": 14847542272, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 155233288192, + "reserved_bytes_current_delta": 23286775808, + "reserved_bytes_peak": 155233288192, + "reserved_bytes_peak_delta": 23286775808, + "segment_count_current": 1097, + "segment_count_current_delta": 169, + "segment_count_peak": 1097, + "segment_count_peak_delta": 169, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.816882Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6913cfb1cc1a230c1cc31ca6a137875410a8fcc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:624ccf54acb5bc1133e019f5644605beeb1cdb228b6a43a6a0670036ce80281d +size 4761905 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b6b9c392a97136479555164476930bd45ecd873 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505479680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130565839872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42505479680, + "active_bytes_peak": 130565839872, + "allocated_bytes_current": 42505479680, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_peak": 46669695488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 155233288192, + "reserved_bytes_peak": 155233288192, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:04:06.418658Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505479680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447068160, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42505479680, + "active_bytes_peak": 130565839872, + "allocated_bytes_current": 42505479680, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_peak": 46669695488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 155233288192, + "reserved_bytes_peak": 155233288192, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:04:05.829885Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46669695488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 155233288192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155233288192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1097, + "segment_count_current_delta": 0, + "segment_count_peak": 1097, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.429246Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac263c70487972eb7ff50afdec5e13d394a38bc2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01890f6d738e8cc73fafbc7c5b378623fb52a98b01996eeb5a9055b6de5cf06f +size 403620 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24251e321e6a9758ecc0be2cf9e0d35f6adcfcc0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42505479680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130565839872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42505479680, + "active_bytes_peak": 130565839872, + "allocated_bytes_current": 42505479680, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_peak": 46669695488, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 155233288192, + "reserved_bytes_peak": 155233288192, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:04:07.024063Z", + "world_size": 8 + }, + "allocated_bytes_current": 42505479680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127447068160, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42505479680, + "active_bytes_peak": 130565839872, + "allocated_bytes_current": 42505479680, + "allocated_bytes_peak": 127447068160, + "device": 5, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_peak": 46669695488, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 155233288192, + "reserved_bytes_peak": 155233288192, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:04:06.430320Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 20530715136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46669695488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 155233288192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155233288192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1097, + "segment_count_current_delta": 0, + "segment_count_peak": 1097, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.033441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3123d5568d83965852b67703c48cbd4f4edde8d0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28828d5ebd6237774c5b03e0400603dab08621171960bff1a06def72d36e5682 +size 403620 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab26f9a155d4ce86833f6456130dcbf64356e2fc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42485283328, + "active_bytes_current_delta": 19439536128, + "active_bytes_peak": 113234178048, + "active_bytes_peak_delta": 90175847936, + "after": { + "active_bytes_current": 42485283328, + "active_bytes_peak": 113234178048, + "allocated_bytes_current": 42485283328, + "allocated_bytes_peak": 113234178048, + "device": 6, + "inactive_split_bytes_current": 18608948736, + "inactive_split_bytes_peak": 30468116992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117549563904, + "reserved_bytes_peak": 117549563904, + "segment_count_current": 926, + "segment_count_peak": 926, + "timestamp_utc": "2026-08-26T19:04:02.079986Z", + "world_size": 8 + }, + "allocated_bytes_current": 42485283328, + "allocated_bytes_current_delta": 19439536128, + "allocated_bytes_peak": 113234178048, + "allocated_bytes_peak_delta": 90175847936, + "before": { + "active_bytes_current": 23045747200, + "active_bytes_peak": 23058330112, + "allocated_bytes_current": 23045747200, + "allocated_bytes_peak": 23058330112, + "device": 6, + "inactive_split_bytes_current": 358469120, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25761415168, + "reserved_bytes_peak": 25761415168, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-26T19:03:23.745539Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18608948736, + "inactive_split_bytes_current_delta": 18250479616, + "inactive_split_bytes_peak": 30468116992, + "inactive_split_bytes_peak_delta": 29584881664, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117549563904, + "reserved_bytes_current_delta": 91788148736, + "reserved_bytes_peak": 117549563904, + "reserved_bytes_peak_delta": 91788148736, + "segment_count_current": 926, + "segment_count_current_delta": 369, + "segment_count_peak": 926, + "segment_count_peak_delta": 369, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.162223Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b209b229607a0d7a06b43ee39d50a54f2b03c04 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5087269156536422fe460c092ada4e54ccaa027b0479bb29957631815242864 +size 4119126 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.json new file mode 100644 index 0000000000000000000000000000000000000000..62ffed606084d73ef24b07373f71e83a8fba6bc4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42485283328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127602459136, + "active_bytes_peak_delta": 14368281088, + "after": { + "active_bytes_current": 42485283328, + "active_bytes_peak": 127602459136, + "allocated_bytes_current": 42485283328, + "allocated_bytes_peak": 127464047104, + "device": 6, + "inactive_split_bytes_current": 18701223424, + "inactive_split_bytes_peak": 30468116992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132095410176, + "reserved_bytes_peak": 132095410176, + "segment_count_current": 939, + "segment_count_peak": 939, + "timestamp_utc": "2026-08-26T19:04:02.816227Z", + "world_size": 8 + }, + "allocated_bytes_current": 42485283328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127464047104, + "allocated_bytes_peak_delta": 14229869056, + "before": { + "active_bytes_current": 42485283328, + "active_bytes_peak": 113234178048, + "allocated_bytes_current": 42485283328, + "allocated_bytes_peak": 113234178048, + "device": 6, + "inactive_split_bytes_current": 18608948736, + "inactive_split_bytes_peak": 30468116992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117549563904, + "reserved_bytes_peak": 117549563904, + "segment_count_current": 926, + "segment_count_peak": 926, + "timestamp_utc": "2026-08-26T19:04:02.174428Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18701223424, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 30468116992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132095410176, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 132095410176, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 939, + "segment_count_current_delta": 13, + "segment_count_peak": 939, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.893379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4512e0cbc5ff681a98560033fe9108ec37358c04 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a6f365f5a7ce3398fb05d1c6cf8ae6aa3ecdc3210fe7c03d685d4a34d72590b +size 4509200 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44c9a471f1f6ff5c00a83eb469683d02c2ea76b6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42485283328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127741936128, + "active_bytes_peak_delta": 139476992, + "after": { + "active_bytes_current": 42485283328, + "active_bytes_peak": 127741936128, + "allocated_bytes_current": 42485283328, + "allocated_bytes_peak": 127465112064, + "device": 6, + "inactive_split_bytes_current": 18703320576, + "inactive_split_bytes_peak": 32325633536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132187684864, + "reserved_bytes_peak": 132187684864, + "segment_count_current": 940, + "segment_count_peak": 940, + "timestamp_utc": "2026-08-26T19:04:03.544749Z", + "world_size": 8 + }, + "allocated_bytes_current": 42485283328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127465112064, + "allocated_bytes_peak_delta": 1064960, + "before": { + "active_bytes_current": 42485283328, + "active_bytes_peak": 127602459136, + "allocated_bytes_current": 42485283328, + "allocated_bytes_peak": 127464047104, + "device": 6, + "inactive_split_bytes_current": 18701223424, + "inactive_split_bytes_peak": 30468116992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132095410176, + "reserved_bytes_peak": 132095410176, + "segment_count_current": 939, + "segment_count_peak": 939, + "timestamp_utc": "2026-08-26T19:04:02.903317Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 18703320576, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 32325633536, + "inactive_split_bytes_peak_delta": 1857516544, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132187684864, + "reserved_bytes_current_delta": 92274688, + "reserved_bytes_peak": 132187684864, + "reserved_bytes_peak_delta": 92274688, + "segment_count_current": 940, + "segment_count_current_delta": 1, + "segment_count_peak": 940, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.620155Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14816370d613f0acc4dc90185d426cae30156a1d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa49457f319d10134dd62b75318b0f7766bf65ef9e67578bb76f120c9311583e +size 4526558 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6238e68b507dd9f3a08dc95e446d6fd7a473da13 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42503244288, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130563604480, + "active_bytes_peak_delta": 2821668352, + "after": { + "active_bytes_current": 42503244288, + "active_bytes_peak": 130563604480, + "allocated_bytes_current": 42503244288, + "allocated_bytes_peak": 127465112064, + "device": 6, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_peak": 46994892288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 155463974912, + "reserved_bytes_peak": 155463974912, + "segment_count_current": 1147, + "segment_count_peak": 1147, + "timestamp_utc": "2026-08-26T19:04:05.729858Z", + "world_size": 8 + }, + "allocated_bytes_current": 42503244288, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127465112064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42485283328, + "active_bytes_peak": 127741936128, + "allocated_bytes_current": 42485283328, + "allocated_bytes_peak": 127465112064, + "device": 6, + "inactive_split_bytes_current": 18703320576, + "inactive_split_bytes_peak": 32325633536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132187684864, + "reserved_bytes_peak": 132187684864, + "segment_count_current": 940, + "segment_count_peak": 940, + "timestamp_utc": "2026-08-26T19:04:03.629844Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_current_delta": 2060316672, + "inactive_split_bytes_peak": 46994892288, + "inactive_split_bytes_peak_delta": 14669258752, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 155463974912, + "reserved_bytes_current_delta": 23276290048, + "reserved_bytes_peak": 155463974912, + "reserved_bytes_peak_delta": 23276290048, + "segment_count_current": 1147, + "segment_count_current_delta": 207, + "segment_count_peak": 1147, + "segment_count_peak_delta": 207, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.816618Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0508a7acc6034020dd7a0e10fea3925479bbbd03 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a24fc4e9476ffa47c1f1f2f9e278b6aa303a80de5d839eda5cb01d2d47d409c +size 4743045 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..409623656ed292dc45450f9b43e0a16e48e1161e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42503244288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130563604480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42503244288, + "active_bytes_peak": 130563604480, + "allocated_bytes_current": 42503244288, + "allocated_bytes_peak": 127465112064, + "device": 6, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_peak": 46994892288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 155463974912, + "reserved_bytes_peak": 155463974912, + "segment_count_current": 1147, + "segment_count_peak": 1147, + "timestamp_utc": "2026-08-26T19:04:06.418670Z", + "world_size": 8 + }, + "allocated_bytes_current": 42503244288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127465112064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42503244288, + "active_bytes_peak": 130563604480, + "allocated_bytes_current": 42503244288, + "allocated_bytes_peak": 127465112064, + "device": 6, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_peak": 46994892288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 155463974912, + "reserved_bytes_peak": 155463974912, + "segment_count_current": 1147, + "segment_count_peak": 1147, + "timestamp_utc": "2026-08-26T19:04:05.827858Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46994892288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 155463974912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155463974912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1147, + "segment_count_current_delta": 0, + "segment_count_peak": 1147, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.429585Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..512563324829acf51d5c3c439383326034547406 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aebbfe8d86a9a148c7944cf029e67444944214d147c175352f0f429eb5660d46 +size 384967 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6065ae96dcd163c7fe253d22ff9e6ca1fd3127b4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42503244288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130563604480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42503244288, + "active_bytes_peak": 130563604480, + "allocated_bytes_current": 42503244288, + "allocated_bytes_peak": 127465112064, + "device": 6, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_peak": 46994892288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 155463974912, + "reserved_bytes_peak": 155463974912, + "segment_count_current": 1147, + "segment_count_peak": 1147, + "timestamp_utc": "2026-08-26T19:04:07.024063Z", + "world_size": 8 + }, + "allocated_bytes_current": 42503244288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127465112064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42503244288, + "active_bytes_peak": 130563604480, + "allocated_bytes_current": 42503244288, + "allocated_bytes_peak": 127465112064, + "device": 6, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_peak": 46994892288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 155463974912, + "reserved_bytes_peak": 155463974912, + "segment_count_current": 1147, + "segment_count_peak": 1147, + "timestamp_utc": "2026-08-26T19:04:06.430686Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 20763637248, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 46994892288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 155463974912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155463974912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1147, + "segment_count_current_delta": 0, + "segment_count_peak": 1147, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.033945Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01b607874dfbee6ca6f24564a3b1ab76f5287967 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd7f049c76c68f9bcc677e416ff783ce76c3ac862ca0efeb7dc803661ca50f8d +size 384967 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb2ab333340cc845d986970227866970b3e8b5ad --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42481551360, + "active_bytes_current_delta": 19433245696, + "active_bytes_peak": 113246329856, + "active_bytes_peak_delta": 90185441280, + "after": { + "active_bytes_current": 42481551360, + "active_bytes_peak": 113246329856, + "allocated_bytes_current": 42481551360, + "allocated_bytes_peak": 113246329856, + "device": 7, + "inactive_split_bytes_current": 18811910144, + "inactive_split_bytes_peak": 30718459904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117838970880, + "reserved_bytes_peak": 117838970880, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:04:02.080054Z", + "world_size": 8 + }, + "allocated_bytes_current": 42481551360, + "allocated_bytes_current_delta": 19433245696, + "allocated_bytes_peak": 113246329856, + "allocated_bytes_peak_delta": 90185441280, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 7, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:03:23.745600Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18811910144, + "inactive_split_bytes_current_delta": 18309198848, + "inactive_split_bytes_peak": 30718459904, + "inactive_split_bytes_peak_delta": 29835224576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117838970880, + "reserved_bytes_current_delta": 92054487040, + "reserved_bytes_peak": 117838970880, + "reserved_bytes_peak_delta": 92054487040, + "segment_count_current": 927, + "segment_count_current_delta": 369, + "segment_count_peak": 927, + "segment_count_peak_delta": 369, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.163072Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba945826878d23052e8d4cb984f3978f060bfa4f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a173ef4ffb81f29a415ae4991c1534810f7f11a29159ed7ba8526ced8aed8e27 +size 4131448 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd2547b1a565475f830c9807cd04119ccd9631d7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42481551360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127725126656, + "active_bytes_peak_delta": 14478796800, + "after": { + "active_bytes_current": 42481551360, + "active_bytes_peak": 127725126656, + "allocated_bytes_current": 42481551360, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 18906281984, + "inactive_split_bytes_peak": 31017596928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 130228944896, + "reserved_bytes_peak": 130228944896, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:02.816197Z", + "world_size": 8 + }, + "allocated_bytes_current": 42481551360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127448441856, + "allocated_bytes_peak_delta": 14202112000, + "before": { + "active_bytes_current": 42481551360, + "active_bytes_peak": 113246329856, + "allocated_bytes_current": 42481551360, + "allocated_bytes_peak": 113246329856, + "device": 7, + "inactive_split_bytes_current": 18811910144, + "inactive_split_bytes_peak": 30718459904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117838970880, + "reserved_bytes_peak": 117838970880, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-26T19:04:02.175428Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18906281984, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 31017596928, + "inactive_split_bytes_peak_delta": 299137024, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 130228944896, + "reserved_bytes_current_delta": 12389974016, + "reserved_bytes_peak": 130228944896, + "reserved_bytes_peak_delta": 12389974016, + "segment_count_current": 935, + "segment_count_current_delta": 8, + "segment_count_peak": 935, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle", + "timestamp_utc": "2026-08-26T19:04:02.893988Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..10c20281da255a4d221da32d2447aed123372bb0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd0e6496b8c4f5e5a893295cce203c7b318f3b559a9b814981578f0119c3b48a +size 4526765 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b7378bfd2d5f4fb05f68fc2f1ee5dccf94efd64 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42481551360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 127725126656, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42481551360, + "active_bytes_peak": 127725126656, + "allocated_bytes_current": 42481551360, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 18904184832, + "inactive_split_bytes_peak": 31017596928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 130321219584, + "reserved_bytes_peak": 130321219584, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:04:03.544788Z", + "world_size": 8 + }, + "allocated_bytes_current": 42481551360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127448441856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42481551360, + "active_bytes_peak": 127725126656, + "allocated_bytes_current": 42481551360, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 18906281984, + "inactive_split_bytes_peak": 31017596928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 130228944896, + "reserved_bytes_peak": 130228944896, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-26T19:04:02.904143Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18904184832, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 31017596928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 130321219584, + "reserved_bytes_current_delta": 92274688, + "reserved_bytes_peak": 130321219584, + "reserved_bytes_peak_delta": 92274688, + "segment_count_current": 936, + "segment_count_current_delta": 1, + "segment_count_peak": 936, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle", + "timestamp_utc": "2026-08-26T19:04:03.621671Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0a6aabba2fe3f9a668e6e80ece40b595fc4568d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b186e0e3517c8a4ba9a3e4dca7e14ae4fc1117cc85b8233f13e6536b0f26ff3c +size 4545022 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21ab9b9e2f1c73d0ce2a5a6b151f1c95614438ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42499512320, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 130559872512, + "active_bytes_peak_delta": 2834745856, + "after": { + "active_bytes_current": 42499512320, + "active_bytes_peak": 130559872512, + "allocated_bytes_current": 42499512320, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_peak": 47197853696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:05.729875Z", + "world_size": 8 + }, + "allocated_bytes_current": 42499512320, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 127448441856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42481551360, + "active_bytes_peak": 127725126656, + "allocated_bytes_current": 42481551360, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 18904184832, + "inactive_split_bytes_peak": 31017596928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 130321219584, + "reserved_bytes_peak": 130321219584, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-26T19:04:03.631572Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_current_delta": 2062413824, + "inactive_split_bytes_peak": 47197853696, + "inactive_split_bytes_peak_delta": 16180256768, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 155661107200, + "reserved_bytes_current_delta": 25339887616, + "reserved_bytes_peak": 155661107200, + "reserved_bytes_peak_delta": 25339887616, + "segment_count_current": 1104, + "segment_count_current_delta": 168, + "segment_count_peak": 1104, + "segment_count_peak_delta": 168, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle", + "timestamp_utc": "2026-08-26T19:04:05.818717Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db6217c4c0327cdad1e3d93e3f8e5f52d639fb72 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e2a654dca35dfc9651a940d33d0c4f97e84e261cec94d4caed1e3bfe091e040 +size 4761004 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7514eb878e3961bd6e86b31905f9061dd814ba87 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42499512320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130559872512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42499512320, + "active_bytes_peak": 130559872512, + "allocated_bytes_current": 42499512320, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_peak": 47197853696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:06.418730Z", + "world_size": 8 + }, + "allocated_bytes_current": 42499512320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127448441856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42499512320, + "active_bytes_peak": 130559872512, + "allocated_bytes_current": 42499512320, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_peak": 47197853696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:05.830255Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 47197853696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 155661107200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155661107200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1104, + "segment_count_current_delta": 0, + "segment_count_peak": 1104, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle", + "timestamp_utc": "2026-08-26T19:04:06.429430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a01820318d659a70942f377ae278d2f6fd69edb3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73415e2e740862df2997998353ec237668184603fe0563123d817019e4d518b7 +size 402457 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a54878c8cf592f1338fa3458714a88a3350758a2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42499512320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 130559872512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42499512320, + "active_bytes_peak": 130559872512, + "allocated_bytes_current": 42499512320, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_peak": 47197853696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:07.024118Z", + "world_size": 8 + }, + "allocated_bytes_current": 42499512320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 127448441856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42499512320, + "active_bytes_peak": 130559872512, + "allocated_bytes_current": 42499512320, + "allocated_bytes_peak": 127448441856, + "device": 7, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_peak": 47197853696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 155661107200, + "reserved_bytes_peak": 155661107200, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:04:06.430521Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 20966598656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 47197853696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 155661107200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 155661107200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1104, + "segment_count_current_delta": 0, + "segment_count_peak": 1104, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle", + "timestamp_utc": "2026-08-26T19:04:07.033515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2373886ad4a55cca43d7ba489359ec31a9a090d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a0671291f458477c03377803b0d7950ec7597b71f5fde4da7621bb7edcce385 +size 402457 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..26bca89b0d5c12e4c50c7fe030b6c2761253edb0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3e2a7495377dfcbe69c788dfd19e1d196c0dada4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log @@ -0,0 +1,1605 @@ +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 19:02:58.534000 602045 torch/distributed/run.py:766] +W0826 19:02:58.534000 602045 torch/distributed/run.py:766] ***************************************** +W0826 19:02:58.534000 602045 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:02:58.534000 602045 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 7546 is connected to is connected to is connected to is connected to 7777 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 +77 + + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 41 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 3 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W826 19:03:17.195792894 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:03:17.197980243 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:03:17.200148645 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:03:17.200226315 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:03:17.200382476 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:03:17.200465539 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:03:17.202024029 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:03:17.352810841 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.691 seconds +time to initialize megatron (seconds): 6.913 +[after megatron is initialized] datetime: 2026-08-26 19:03:22 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:03:23 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:03:23 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (625.44, 632.05) + train/valid/test-data-iterators-setup ..........: (64.41, 72.55) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:03:23 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:04:02] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 38335.0 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.620 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 40504.5634765625 | max allocated: 107989.0107421875 | reserved: 112038.0 | max reserved: 112038.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:04:02] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 736.8 | throughput per GPU (TFLOP/s/GPU): 173.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.566 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:04:03] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 728.5 | throughput per GPU (TFLOP/s/GPU): 175.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952471E+00 | loss scale: 1.0 | grad norm: 42.237 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:04:05] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2184.7 | throughput per GPU (TFLOP/s/GPU): 58.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476898E+00 | loss scale: 1.0 | grad norm: 7.154 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:04:06] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 689.1 | throughput per GPU (TFLOP/s/GPU): 185.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.114793E+01 | loss scale: 1.0 | grad norm: 400.648 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:04:07] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 605.3 | throughput per GPU (TFLOP/s/GPU): 210.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.368878E+00 | loss scale: 1.0 | grad norm: 5.139 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:04:07 +W0826 19:04:07.220000 602045 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:04:07.221000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602134 closing signal SIGTERM +W0826 19:04:07.222000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602135 closing signal SIGTERM +W0826 19:04:07.224000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602136 closing signal SIGTERM +W0826 19:04:07.227000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602137 closing signal SIGTERM +W0826 19:04:07.229000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602138 closing signal SIGTERM +W0826 19:04:07.234000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602139 closing signal SIGTERM +W0826 19:04:07.237000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602140 closing signal SIGTERM +W0826 19:04:07.239000 602045 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 602141 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 602045 got signal: 15 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..0b293791dc74569b0fc6089ed365389684f59def --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 801 +1, 783 +2, 795 +3, 797 +4, 795 +5, 783 +6, 773 +7, 781 +0, 1077 +1, 1065 +2, 1077 +3, 1073 +4, 1069 +5, 1077 +6, 1077 +7, 1077 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 26289 +1, 26377 +2, 26223 +3, 26223 +4, 26223 +5, 26223 +6, 26355 +7, 26377 +0, 32143 +1, 32229 +2, 32079 +3, 32077 +4, 32077 +5, 32075 +6, 32209 +7, 32229 +0, 36899 +1, 36983 +2, 36833 +3, 36831 +4, 36831 +5, 36829 +6, 36965 +7, 36983 +0, 37783 +1, 37867 +2, 37717 +3, 37715 +4, 37715 +5, 37713 +6, 37849 +7, 37867 +0, 37871 +1, 37955 +2, 37805 +3, 37803 +4, 37803 +5, 37801 +6, 37937 +7, 37955 +0, 38073 +1, 38151 +2, 38041 +3, 38047 +4, 37983 +5, 38009 +6, 38147 +7, 38167 +0, 38819 +1, 38903 +2, 38753 +3, 38753 +4, 38753 +5, 38749 +6, 38885 +7, 38903 +0, 38819 +1, 38903 +2, 38753 +3, 38753 +4, 38753 +5, 38749 +6, 38885 +7, 38903 +0, 38827 +1, 38911 +2, 38761 +3, 38761 +4, 38761 +5, 38757 +6, 38893 +7, 38911 +0, 38829 +1, 38915 +2, 38765 +3, 38763 +4, 38763 +5, 38761 +6, 38895 +7, 38915 +0, 41073 +1, 41157 +2, 41011 +3, 41007 +4, 41009 +5, 41011 +6, 41145 +7, 41157 +0, 111971 +1, 112009 +2, 111911 +3, 111867 +4, 111851 +5, 111885 +6, 111979 +7, 112019 +0, 112445 +1, 112531 +2, 112381 +3, 112379 +4, 112379 +5, 112377 +6, 112511 +7, 112531 +0, 112445 +1, 112531 +2, 112381 +3, 112379 +4, 112379 +5, 112377 +6, 112511 +7, 112531 +0, 112445 +1, 112531 +2, 112381 +3, 112379 +4, 112379 +5, 112377 +6, 112511 +7, 112531 +0, 112445 +1, 112531 +2, 112381 +3, 112379 +4, 112379 +5, 112377 +6, 112511 +7, 112531 +0, 112897 +1, 112965 +2, 112911 +3, 112889 +4, 112853 +5, 112861 +6, 112965 +7, 112937 +0, 115059 +1, 115145 +2, 114995 +3, 114993 +4, 115433 +5, 115167 +6, 115125 +7, 115145 +0, 115685 +1, 115823 +2, 115647 +3, 115671 +4, 115627 +5, 115631 +6, 115729 +7, 115781 +0, 116503 +1, 116567 +2, 116471 +3, 116469 +4, 116459 +5, 116407 +6, 116549 +7, 116531 +0, 130797 +1, 128829 +2, 130733 +3, 130731 +4, 128683 +5, 130723 +6, 130863 +7, 130877 +0, 130797 +1, 128829 +2, 130733 +3, 130731 +4, 128683 +5, 130723 +6, 130863 +7, 130877 +0, 130797 +1, 128829 +2, 130733 +3, 130731 +4, 128683 +5, 130723 +6, 130863 +7, 130877 +0, 777 +1, 1034 +2, 129785 +3, 2058 +4, 778 +5, 778 +6, 778 +7, 778 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190523170489Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190523170489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b86e2bd70cb84afaa19b197a1ba30aa43748c465 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190523170489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534091776, + "active_bytes_current_delta": 19488525824, + "active_bytes_peak": 110917190656, + "active_bytes_peak_delta": 87859041792, + "after": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 110917190656, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 110917190656, + "device": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_peak": 28251957248, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115114770432, + "reserved_bytes_peak": 115114770432, + "segment_count_current": 913, + "segment_count_peak": 913, + "timestamp_utc": "2026-08-26T19:05:23.169988Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534091776, + "allocated_bytes_current_delta": 19488525824, + "allocated_bytes_peak": 110917190656, + "allocated_bytes_peak_delta": 87859041792, + "before": { + "active_bytes_current": 23045565952, + "active_bytes_peak": 23058148864, + "allocated_bytes_current": 23045565952, + "allocated_bytes_peak": 23058148864, + "device": 0, + "inactive_split_bytes_current": 352358912, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 25692209152, + "reserved_bytes_peak": 25692209152, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-26T19:04:39.309298Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_current_delta": 15978508800, + "inactive_split_bytes_peak": 28251957248, + "inactive_split_bytes_peak_delta": 27368721920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115114770432, + "reserved_bytes_current_delta": 89422561280, + "reserved_bytes_peak": 115114770432, + "reserved_bytes_peak_delta": 89422561280, + "segment_count_current": 913, + "segment_count_current_delta": 359, + "segment_count_peak": 913, + "segment_count_peak_delta": 359, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190523170489Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.256613Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190523170489Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190523170489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6b95cdbb790724200d4c9c082a2a78bfeae2ebb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190523170489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41767fe5aaff30c6281eec208a60ee19b13fb2da6185880ae841362676638384 +size 4109022 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190524116017Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190524116017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c63bec0c573147abcc32a5334cb0802c7e7c153b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190524116017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534091776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125107016704, + "active_bytes_peak_delta": 14189826048, + "after": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 125107016704, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 125107016704, + "device": 0, + "inactive_split_bytes_current": 16238593024, + "inactive_split_bytes_peak": 28251957248, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_peak": 129658519552, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:24.115569Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534091776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125107016704, + "allocated_bytes_peak_delta": 14189826048, + "before": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 110917190656, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 110917190656, + "device": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_peak": 28251957248, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115114770432, + "reserved_bytes_peak": 115114770432, + "segment_count_current": 913, + "segment_count_peak": 913, + "timestamp_utc": "2026-08-26T19:05:23.268787Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16238593024, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28251957248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 129658519552, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 925, + "segment_count_current_delta": 12, + "segment_count_peak": 925, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190524116017Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.196175Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190524116017Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190524116017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d015f65ad38909590d59591c9a3dd3046929eb13 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190524116017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbd61222af38a744d9bd0c7b7520b8de11d61188963fdbdc4da3120c03a757c6 +size 4511253 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190525012064Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190525012064Z.json new file mode 100644 index 0000000000000000000000000000000000000000..720e28582886d101e95767f5a77afb50738c41a4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190525012064Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534091776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125116626944, + "active_bytes_peak_delta": 9610240, + "after": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 125116626944, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 125116626944, + "device": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_peak": 28251957248, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_peak": 129658519552, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.011649Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534091776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125116626944, + "allocated_bytes_peak_delta": 9610240, + "before": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 125107016704, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 125107016704, + "device": 0, + "inactive_split_bytes_current": 16238593024, + "inactive_split_bytes_peak": 28251957248, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_peak": 129658519552, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:24.206342Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28251957248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129658519552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 925, + "segment_count_current_delta": 0, + "segment_count_peak": 925, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190525012064Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.088689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190525012064Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190525012064Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c44ba70658388932bb7548e136282246deadda6d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190525012064Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce0e33c0ec4a485ee8a4cfa46edfacba527462dabe5e92d2f83ef20ec0a47b88 +size 4529312 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190525902106Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190525902106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27ae9c09ba6973424e3826190dfe5e63c74d5dc5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190525902106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534091776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125116626944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 125116626944, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 125116626944, + "device": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_peak": 28251957248, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_peak": 129658519552, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.901694Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534091776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125116626944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 125116626944, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 125116626944, + "device": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_peak": 28251957248, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_peak": 129658519552, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.098416Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28251957248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129658519552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 925, + "segment_count_current_delta": 0, + "segment_count_peak": 925, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190525902106Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.983859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190525902106Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190525902106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5b39c9a16a7978294b23933448e37c2f57f52d8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190525902106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5ed7ef0d75ef52e9b94c0ab72930bb8302e3aa205ed1f79fd10c6394c32aa99 +size 4529237 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190526795562Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190526795562Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b1ed01ae6f8c698ad45011faa2afa0543780840 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190526795562Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534091776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125116626944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 125116626944, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 125116626944, + "device": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_peak": 28251957248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_peak": 129658519552, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:26.795152Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534091776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125116626944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42534091776, + "active_bytes_peak": 125116626944, + "allocated_bytes_current": 42534091776, + "allocated_bytes_peak": 125116626944, + "device": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_peak": 28251957248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_peak": 129658519552, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.995662Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16330867712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28251957248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 129658519552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129658519552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 925, + "segment_count_current_delta": 0, + "segment_count_peak": 925, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190526795562Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.868222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190526795562Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190526795562Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da14f0e02bd843b848971f8b2fbb922f3d66b443 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190526795562Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbd2588bc04aadef7367ef47fa8fb3f7e0d449e2006203fc3241cd9cf34e1d4a +size 4529327 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190523170722Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190523170722Z.json new file mode 100644 index 0000000000000000000000000000000000000000..395ec3714589f4ef67141e1a047e0e1018f60354 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190523170722Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42527216128, + "active_bytes_current_delta": 19478910464, + "active_bytes_peak": 110915870720, + "active_bytes_peak_delta": 87854982144, + "after": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 110915870720, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 110915870720, + "device": 1, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_peak": 28333716480, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115204947968, + "reserved_bytes_peak": 115204947968, + "segment_count_current": 916, + "segment_count_peak": 916, + "timestamp_utc": "2026-08-26T19:05:23.170221Z", + "world_size": 8 + }, + "allocated_bytes_current": 42527216128, + "allocated_bytes_current_delta": 19478910464, + "allocated_bytes_peak": 110915870720, + "allocated_bytes_peak_delta": 87854982144, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 1, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:04:39.309169Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_current_delta": 15891655168, + "inactive_split_bytes_peak": 28333716480, + "inactive_split_bytes_peak_delta": 27450481152, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115204947968, + "reserved_bytes_current_delta": 89420464128, + "reserved_bytes_peak": 115204947968, + "reserved_bytes_peak_delta": 89420464128, + "segment_count_current": 916, + "segment_count_current_delta": 358, + "segment_count_peak": 916, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190523170722Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.253352Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190523170722Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190523170722Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7bb0bdba1e3e8a2a94c209755c89caa9ed18b55e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190523170722Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb9f2bf1dc1efcf132644703c860817418c70a96c21f4913896a6123f011167b +size 4048565 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190524115917Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190524115917Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60947195013691938147229db3855eecf2d515f7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190524115917Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42527216128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125076103168, + "active_bytes_peak_delta": 14160232448, + "after": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 125076103168, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 125076103168, + "device": 1, + "inactive_split_bytes_current": 16302091776, + "inactive_split_bytes_peak": 28333716480, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127594921984, + "reserved_bytes_peak": 127594921984, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-26T19:05:24.115492Z", + "world_size": 8 + }, + "allocated_bytes_current": 42527216128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125076103168, + "allocated_bytes_peak_delta": 14160232448, + "before": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 110915870720, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 110915870720, + "device": 1, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_peak": 28333716480, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115204947968, + "reserved_bytes_peak": 115204947968, + "segment_count_current": 916, + "segment_count_peak": 916, + "timestamp_utc": "2026-08-26T19:05:23.264335Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16302091776, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28333716480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127594921984, + "reserved_bytes_current_delta": 12389974016, + "reserved_bytes_peak": 127594921984, + "reserved_bytes_peak_delta": 12389974016, + "segment_count_current": 924, + "segment_count_current_delta": 8, + "segment_count_peak": 924, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190524115917Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.192998Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190524115917Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190524115917Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce95077a8e6672ea333ac1cf1b1a8414764c9cb9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190524115917Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0afbb29fdd24795bd61781252ff1cad1fcd5292c44bf044425eb764b46bd236 +size 4451763 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190525012099Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190525012099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6397791b4dbc3ab03c49cfad089daea175d314b4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190525012099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42527216128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125089795584, + "active_bytes_peak_delta": 13692416, + "after": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 125089795584, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 125089795584, + "device": 1, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_peak": 28333716480, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127594921984, + "reserved_bytes_peak": 127594921984, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-26T19:05:25.011659Z", + "world_size": 8 + }, + "allocated_bytes_current": 42527216128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125089795584, + "allocated_bytes_peak_delta": 13692416, + "before": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 125076103168, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 125076103168, + "device": 1, + "inactive_split_bytes_current": 16302091776, + "inactive_split_bytes_peak": 28333716480, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127594921984, + "reserved_bytes_peak": 127594921984, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-26T19:05:24.201776Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28333716480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127594921984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127594921984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 924, + "segment_count_current_delta": 0, + "segment_count_peak": 924, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190525012099Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.086254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190525012099Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190525012099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc2a14e4c16255d7d113861399e070b2c212f4ca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190525012099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c79950214c7ff17e3d21df7c5e4506efb8005c4bff76118b65eb759e0a48ef1b +size 4469430 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190525902066Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190525902066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ad4886566b32103885bd73b8feb27364b36422f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190525902066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42527216128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125089795584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 125089795584, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 125089795584, + "device": 1, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_peak": 28333716480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127594921984, + "reserved_bytes_peak": 127594921984, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-26T19:05:25.901658Z", + "world_size": 8 + }, + "allocated_bytes_current": 42527216128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125089795584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 125089795584, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 125089795584, + "device": 1, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_peak": 28333716480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127594921984, + "reserved_bytes_peak": 127594921984, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-26T19:05:25.095064Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28333716480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127594921984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127594921984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 924, + "segment_count_current_delta": 0, + "segment_count_peak": 924, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190525902066Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.973677Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190525902066Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190525902066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e173ab3039c2f41b209c7e54be235a8f57d26d2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190525902066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d206e1301af73c54a32dc1c52f3156b4b1671b13344a3a831180416b3d151c43 +size 4469635 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190526795585Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190526795585Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6cd2717d4b3715970582dc8e48d4d085f0e3dc48 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190526795585Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42527216128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125089795584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 125089795584, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 125089795584, + "device": 1, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_peak": 28333716480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127594921984, + "reserved_bytes_peak": 127594921984, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-26T19:05:26.795177Z", + "world_size": 8 + }, + "allocated_bytes_current": 42527216128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125089795584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42527216128, + "active_bytes_peak": 125089795584, + "allocated_bytes_current": 42527216128, + "allocated_bytes_peak": 125089795584, + "device": 1, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_peak": 28333716480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127594921984, + "reserved_bytes_peak": 127594921984, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-26T19:05:25.981954Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16394366464, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28333716480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127594921984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127594921984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 924, + "segment_count_current_delta": 0, + "segment_count_peak": 924, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190526795585Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.865289Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190526795585Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190526795585Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2358d6786c0ff9caaa3a467c90983144266f827e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190526795585Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:565bc4aaf33f00f6e2c9b1b86f95dfcfd863ab8d2078b56d319712205fc3df6e +size 4469679 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190523170333Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190523170333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..631c1afee80db7e4427a740520dde2f85addd6c1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190523170333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534960128, + "active_bytes_current_delta": 19484090368, + "active_bytes_peak": 110915508224, + "active_bytes_peak_delta": 87852055552, + "after": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 110915508224, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 110915508224, + "device": 2, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_peak": 28099105792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115047661568, + "reserved_bytes_peak": 115047661568, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-26T19:05:23.169853Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534960128, + "allocated_bytes_current_delta": 19484090368, + "allocated_bytes_peak": 110915508224, + "allocated_bytes_peak_delta": 87852055552, + "before": { + "active_bytes_current": 23050869760, + "active_bytes_peak": 23063452672, + "allocated_bytes_current": 23050869760, + "allocated_bytes_peak": 23063452672, + "device": 2, + "inactive_split_bytes_current": 363832320, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:04:39.309357Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_current_delta": 15850823680, + "inactive_split_bytes_peak": 28099105792, + "inactive_split_bytes_peak_delta": 27215870464, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115047661568, + "reserved_bytes_current_delta": 89424658432, + "reserved_bytes_peak": 115047661568, + "reserved_bytes_peak_delta": 89424658432, + "segment_count_current": 911, + "segment_count_current_delta": 360, + "segment_count_peak": 911, + "segment_count_peak_delta": 360, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190523170333Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.252096Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190523170333Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190523170333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1fa55d92f1c1e3f3080b65a1dbc43977ec47d635 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190523170333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e75d04873980563742878f85b8eda9af010d1224ffe3a9b2aa0ca54febe2679 +size 4023318 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190524116015Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190524116015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e92927794253ad082a4b7e771acde80e39fc9d2b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190524116015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534960128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125110946816, + "active_bytes_peak_delta": 14195438592, + "after": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 125110946816, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 125110946816, + "device": 2, + "inactive_split_bytes_current": 16122381312, + "inactive_split_bytes_peak": 28099105792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 129591410688, + "reserved_bytes_peak": 129591410688, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:05:24.115582Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534960128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125110946816, + "allocated_bytes_peak_delta": 14195438592, + "before": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 110915508224, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 110915508224, + "device": 2, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_peak": 28099105792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115047661568, + "reserved_bytes_peak": 115047661568, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-26T19:05:23.264052Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16122381312, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28099105792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 129591410688, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 129591410688, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 923, + "segment_count_current_delta": 12, + "segment_count_peak": 923, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190524116015Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.196707Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190524116015Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190524116015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03ce7112078fe3d29646502cf8047c716a6fd950 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190524116015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e4033f3fc26d26ba1740a94e68f24e67f7479aebcf173b94d96c9c3831e9a26 +size 4423545 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190525012056Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190525012056Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa9c3b129b3e54344a213483112c8a63b266c64f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190525012056Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534960128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125114234368, + "active_bytes_peak_delta": 3287552, + "after": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 125114234368, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 125114234368, + "device": 2, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_peak": 28099105792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 129591410688, + "reserved_bytes_peak": 129591410688, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:05:25.011631Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534960128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125114234368, + "allocated_bytes_peak_delta": 3287552, + "before": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 125110946816, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 125110946816, + "device": 2, + "inactive_split_bytes_current": 16122381312, + "inactive_split_bytes_peak": 28099105792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 129591410688, + "reserved_bytes_peak": 129591410688, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:05:24.206674Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28099105792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 129591410688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129591410688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 923, + "segment_count_current_delta": 0, + "segment_count_peak": 923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190525012056Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.088989Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190525012056Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190525012056Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cba8a4985e0bf6ef270a4ada6c339e424b364720 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190525012056Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:948b49a37c249c5fcf7684b171ff43df587878ee5a45a90e09c6f3a186f38fa1 +size 4441238 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190525902077Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190525902077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71824b8657de6ddcf78a43f4989048e2390342f0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190525902077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534960128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125114234368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 125114234368, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 125114234368, + "device": 2, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_peak": 28099105792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 129591410688, + "reserved_bytes_peak": 129591410688, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:05:25.901667Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534960128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125114234368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 125114234368, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 125114234368, + "device": 2, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_peak": 28099105792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 129591410688, + "reserved_bytes_peak": 129591410688, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:05:25.098880Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28099105792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 129591410688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129591410688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 923, + "segment_count_current_delta": 0, + "segment_count_peak": 923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190525902077Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.977613Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190525902077Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190525902077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e179b56e38403dc6ada1d986641499471b1145d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190525902077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f357968c77311af4a49a18ee088e66a2ffc57df11b25c60269a4f2b7d44c38 +size 4441106 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190526795566Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190526795566Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae733cedbc9a0bd78834d05024f4b386f9f66c38 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190526795566Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534960128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125118711296, + "active_bytes_peak_delta": 4476928, + "after": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 125118711296, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 125118711296, + "device": 2, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_peak": 28099105792, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 129591410688, + "reserved_bytes_peak": 129591410688, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:05:26.795152Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534960128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125118711296, + "allocated_bytes_peak_delta": 4476928, + "before": { + "active_bytes_current": 42534960128, + "active_bytes_peak": 125114234368, + "allocated_bytes_current": 42534960128, + "allocated_bytes_peak": 125114234368, + "device": 2, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_peak": 28099105792, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 129591410688, + "reserved_bytes_peak": 129591410688, + "segment_count_current": 923, + "segment_count_peak": 923, + "timestamp_utc": "2026-08-26T19:05:25.987460Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16214656000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28099105792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 129591410688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129591410688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 923, + "segment_count_current_delta": 0, + "segment_count_peak": 923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190526795566Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.871236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190526795566Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190526795566Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e298e78c9d193825affcbb628ff5e493adca6b4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190526795566Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a110e8dd141792643aa8d35f49192eb021fd433e1f90c79671aa1c8d3839af71 +size 4441175 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190523170333Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190523170333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06277976b5a00393e2b725aaa7e537db49355eab --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190523170333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42526418944, + "active_bytes_current_delta": 19477475840, + "active_bytes_peak": 110902027776, + "active_bytes_peak_delta": 87840501760, + "after": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 110902027776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 110902027776, + "device": 3, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_peak": 28358408704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115045564416, + "reserved_bytes_peak": 115045564416, + "segment_count_current": 910, + "segment_count_peak": 910, + "timestamp_utc": "2026-08-26T19:05:23.169844Z", + "world_size": 8 + }, + "allocated_bytes_current": 42526418944, + "allocated_bytes_current_delta": 19477475840, + "allocated_bytes_peak": 110902027776, + "allocated_bytes_peak_delta": 87840501760, + "before": { + "active_bytes_current": 23048943104, + "active_bytes_peak": 23061526016, + "allocated_bytes_current": 23048943104, + "allocated_bytes_peak": 23061526016, + "device": 3, + "inactive_split_bytes_current": 468519424, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:04:39.309462Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_current_delta": 15807106560, + "inactive_split_bytes_peak": 28358408704, + "inactive_split_bytes_peak_delta": 27475173376, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115045564416, + "reserved_bytes_current_delta": 89422561280, + "reserved_bytes_peak": 115045564416, + "reserved_bytes_peak_delta": 89422561280, + "segment_count_current": 910, + "segment_count_current_delta": 359, + "segment_count_peak": 910, + "segment_count_peak_delta": 359, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190523170333Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.250901Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190523170333Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190523170333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76d13d18a692421594b25813afaa98bf72585633 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190523170333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fc779d7f0a0beeba73915920662fd870510915e6d746118ce9722aad5c17bad +size 4019364 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190524115962Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190524115962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83a4018766468576f8245519500a802f85df21a6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190524115962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42526418944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125080395776, + "active_bytes_peak_delta": 14178368000, + "after": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 125080395776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 125080395776, + "device": 3, + "inactive_split_bytes_current": 16183351296, + "inactive_split_bytes_peak": 28358408704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 129589313536, + "reserved_bytes_peak": 129589313536, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:05:24.115518Z", + "world_size": 8 + }, + "allocated_bytes_current": 42526418944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125080395776, + "allocated_bytes_peak_delta": 14178368000, + "before": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 110902027776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 110902027776, + "device": 3, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_peak": 28358408704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115045564416, + "reserved_bytes_peak": 115045564416, + "segment_count_current": 910, + "segment_count_peak": 910, + "timestamp_utc": "2026-08-26T19:05:23.262671Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16183351296, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28358408704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 129589313536, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 129589313536, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 922, + "segment_count_current_delta": 12, + "segment_count_peak": 922, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190524115962Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.195146Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190524115962Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190524115962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72046fee6f035b5ecdac4e42bbb3318fd113c70f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190524115962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f59d98a3a55941ca212e82bdd5e4b095fca470bebaf647fa3b9553763300930 +size 4449377 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190525012123Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190525012123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ac5183b7b4803317e3a1992ceac6b308ced4fd8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190525012123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42526418944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125080395776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 125080395776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 125080395776, + "device": 3, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_peak": 28358408704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 129589313536, + "reserved_bytes_peak": 129589313536, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:05:25.011707Z", + "world_size": 8 + }, + "allocated_bytes_current": 42526418944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125080395776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 125080395776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 125080395776, + "device": 3, + "inactive_split_bytes_current": 16183351296, + "inactive_split_bytes_peak": 28358408704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 129589313536, + "reserved_bytes_peak": 129589313536, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:05:24.205009Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28358408704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 129589313536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129589313536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 922, + "segment_count_current_delta": 0, + "segment_count_peak": 922, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190525012123Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.086678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190525012123Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190525012123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc499f77d6349f8f669fc37ff49a8d32885577ac --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190525012123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f547d5cf6c0e153c680eb515e4eab544116c90d1b02e2fc7db44d7520e21431 +size 4525022 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190525902070Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190525902070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0721547630c4f2abc36734b9791a5a28391a276 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190525902070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42526418944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125080395776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 125080395776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 125080395776, + "device": 3, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_peak": 28358408704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 129589313536, + "reserved_bytes_peak": 129589313536, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:05:25.901659Z", + "world_size": 8 + }, + "allocated_bytes_current": 42526418944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125080395776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 125080395776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 125080395776, + "device": 3, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_peak": 28358408704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 129589313536, + "reserved_bytes_peak": 129589313536, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:05:25.096072Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28358408704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 129589313536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129589313536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 922, + "segment_count_current_delta": 0, + "segment_count_peak": 922, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190525902070Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.975989Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190525902070Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190525902070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d22643e8812174bbc11501279beb952ff5cacaa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190525902070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1d5e63f8ddb2035be61e80d0bcac00f35314fe6aa2c161eb35febc0ec6824dc +size 4525080 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190526795566Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190526795566Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40d63aed9bbf55d10356855abfccc8f8e4f5cafa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190526795566Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42526418944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125080395776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 125080395776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 125080395776, + "device": 3, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_peak": 28358408704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 129589313536, + "reserved_bytes_peak": 129589313536, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:05:26.795156Z", + "world_size": 8 + }, + "allocated_bytes_current": 42526418944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125080395776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42526418944, + "active_bytes_peak": 125080395776, + "allocated_bytes_current": 42526418944, + "allocated_bytes_peak": 125080395776, + "device": 3, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_peak": 28358408704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 129589313536, + "reserved_bytes_peak": 129589313536, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-26T19:05:25.985501Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16275625984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28358408704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 129589313536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129589313536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 922, + "segment_count_current_delta": 0, + "segment_count_peak": 922, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190526795566Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.868099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190526795566Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190526795566Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a6e5260e73a1d726c1e704bf22950a7a740a685 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190526795566Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7368d37b5b2f0da586d9a4dab88ed2f1c3dd4001ef37b580af9c59c9c1b2a6 +size 4525158 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190523170334Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190523170334Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28d9da5bc91c08646b98e6d0db7dea83cafa623b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190523170334Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42533921792, + "active_bytes_current_delta": 19485538304, + "active_bytes_peak": 110893806592, + "active_bytes_peak_delta": 87832840192, + "after": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 110893806592, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 110893806592, + "device": 4, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_peak": 28242828800, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115045564416, + "reserved_bytes_peak": 115045564416, + "segment_count_current": 910, + "segment_count_peak": 910, + "timestamp_utc": "2026-08-26T19:05:23.169790Z", + "world_size": 8 + }, + "allocated_bytes_current": 42533921792, + "allocated_bytes_current_delta": 19485538304, + "allocated_bytes_peak": 110893806592, + "allocated_bytes_peak_delta": 87832840192, + "before": { + "active_bytes_current": 23048383488, + "active_bytes_peak": 23060966400, + "allocated_bytes_current": 23048383488, + "allocated_bytes_peak": 23060966400, + "device": 4, + "inactive_split_bytes_current": 376804352, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:04:39.309109Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_current_delta": 15889221632, + "inactive_split_bytes_peak": 28242828800, + "inactive_split_bytes_peak_delta": 27359593472, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115045564416, + "reserved_bytes_current_delta": 89422561280, + "reserved_bytes_peak": 115045564416, + "reserved_bytes_peak_delta": 89422561280, + "segment_count_current": 910, + "segment_count_current_delta": 359, + "segment_count_peak": 910, + "segment_count_peak_delta": 359, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190523170334Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.256725Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190523170334Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190523170334Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b63baf65e4cbbba19a048d7b28f5e5e5141cad86 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190523170334Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f2c51b4e9dcfdfd141a165d2dbfbb80ac335ddcb354af46399caa61071e0667 +size 4026488 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190524115970Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190524115970Z.json new file mode 100644 index 0000000000000000000000000000000000000000..512536c27172dc06120b047d01feb630b78918b9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190524115970Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42533921792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125091412992, + "active_bytes_peak_delta": 14197606400, + "after": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 125091412992, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 125091412992, + "device": 4, + "inactive_split_bytes_current": 16173751296, + "inactive_split_bytes_peak": 28242828800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127441829888, + "reserved_bytes_peak": 127441829888, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:05:24.115543Z", + "world_size": 8 + }, + "allocated_bytes_current": 42533921792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125091412992, + "allocated_bytes_peak_delta": 14197606400, + "before": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 110893806592, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 110893806592, + "device": 4, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_peak": 28242828800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115045564416, + "reserved_bytes_peak": 115045564416, + "segment_count_current": 910, + "segment_count_peak": 910, + "timestamp_utc": "2026-08-26T19:05:23.267725Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16173751296, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28242828800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127441829888, + "reserved_bytes_current_delta": 12396265472, + "reserved_bytes_peak": 127441829888, + "reserved_bytes_peak_delta": 12396265472, + "segment_count_current": 921, + "segment_count_current_delta": 11, + "segment_count_peak": 921, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190524115970Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.194783Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190524115970Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190524115970Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a354d7b43fb7f6c885a6d4f910fabc21639780e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190524115970Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61b925544c28bf4c6be7d5389032bc96d68beff4e4d8351cbee025ec5c7ea0bb +size 4426190 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190525012084Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190525012084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..015ca7fc3f8a15f68d0cff2539da2e0f41be8db8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190525012084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42533921792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125105389056, + "active_bytes_peak_delta": 13976064, + "after": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 125105389056, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 125105389056, + "device": 4, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_peak": 28242828800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127441829888, + "reserved_bytes_peak": 127441829888, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:05:25.011677Z", + "world_size": 8 + }, + "allocated_bytes_current": 42533921792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125105389056, + "allocated_bytes_peak_delta": 13976064, + "before": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 125091412992, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 125091412992, + "device": 4, + "inactive_split_bytes_current": 16173751296, + "inactive_split_bytes_peak": 28242828800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127441829888, + "reserved_bytes_peak": 127441829888, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:05:24.203733Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28242828800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127441829888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127441829888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 921, + "segment_count_current_delta": 0, + "segment_count_peak": 921, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190525012084Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.083236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190525012084Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190525012084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc7deb0bd6465e4efaebac2e88352d1ffb03ef5b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190525012084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c14a738320d4f323542c4c994a07f4b89be69c2d5cec795e1a13e8af325f4f +size 4444010 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190525902052Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190525902052Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb86f6e4246cf85f7d85b6a2ae24cd3d43fed350 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190525902052Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42533921792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125105389056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 125105389056, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 125105389056, + "device": 4, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_peak": 28242828800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127441829888, + "reserved_bytes_peak": 127441829888, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:05:25.901639Z", + "world_size": 8 + }, + "allocated_bytes_current": 42533921792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125105389056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 125105389056, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 125105389056, + "device": 4, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_peak": 28242828800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127441829888, + "reserved_bytes_peak": 127441829888, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:05:25.091837Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28242828800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127441829888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127441829888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 921, + "segment_count_current_delta": 0, + "segment_count_peak": 921, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190525902052Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.973573Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190525902052Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190525902052Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..43bfd6d77ae7d718916dbdf92e619e088ff0b506 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190525902052Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb9ebec397601ae8bb12be0f061e544834a8afdafc49564510da336a4b3cbb49 +size 4443945 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190526795553Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190526795553Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0763bc4fb01c36cac6097286ae1659ad65792bf1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190526795553Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42533921792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125106765824, + "active_bytes_peak_delta": 1376768, + "after": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 125106765824, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 125106765824, + "device": 4, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_peak": 28242828800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127441829888, + "reserved_bytes_peak": 127441829888, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:05:26.795138Z", + "world_size": 8 + }, + "allocated_bytes_current": 42533921792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125106765824, + "allocated_bytes_peak_delta": 1376768, + "before": { + "active_bytes_current": 42533921792, + "active_bytes_peak": 125105389056, + "allocated_bytes_current": 42533921792, + "allocated_bytes_peak": 125105389056, + "device": 4, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_peak": 28242828800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127441829888, + "reserved_bytes_peak": 127441829888, + "segment_count_current": 921, + "segment_count_peak": 921, + "timestamp_utc": "2026-08-26T19:05:25.982148Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16266025984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28242828800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127441829888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127441829888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 921, + "segment_count_current_delta": 0, + "segment_count_peak": 921, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190526795553Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.865896Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190526795553Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190526795553Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5daa4174439043ede99b822d1ab8853b327e3f1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190526795553Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ccbffe2b352cc0a19e3adf8e891cbb07eff7d33a39187e8cecf452c46ad25e5 +size 4444012 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190523170322Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190523170322Z.json new file mode 100644 index 0000000000000000000000000000000000000000..206feb9120bfdc1a3051ce0f1d3a279c83ef363d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190523170322Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42536631296, + "active_bytes_current_delta": 19483448320, + "active_bytes_peak": 110900092928, + "active_bytes_peak_delta": 87834327040, + "after": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 110900092928, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 110900092928, + "device": 5, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_peak": 28286038016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115043467264, + "reserved_bytes_peak": 115043467264, + "segment_count_current": 909, + "segment_count_peak": 909, + "timestamp_utc": "2026-08-26T19:05:23.169801Z", + "world_size": 8 + }, + "allocated_bytes_current": 42536631296, + "allocated_bytes_current_delta": 19483448320, + "allocated_bytes_peak": 110900092928, + "allocated_bytes_peak_delta": 87834327040, + "before": { + "active_bytes_current": 23053182976, + "active_bytes_peak": 23065765888, + "allocated_bytes_current": 23053182976, + "allocated_bytes_peak": 23065765888, + "device": 5, + "inactive_split_bytes_current": 462182400, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:04:39.309109Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_current_delta": 15761288192, + "inactive_split_bytes_peak": 28286038016, + "inactive_split_bytes_peak_delta": 27402802688, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115043467264, + "reserved_bytes_current_delta": 89420464128, + "reserved_bytes_peak": 115043467264, + "reserved_bytes_peak_delta": 89420464128, + "segment_count_current": 909, + "segment_count_current_delta": 358, + "segment_count_peak": 909, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190523170322Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.250093Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190523170322Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190523170322Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e84cc4756f7d355591c9cd9bef44b2c5cb6282f7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190523170322Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71718855c193a50858ff4d696b924d2ff669328278f265094cde7e82a419d8f +size 4046288 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190524115966Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190524115966Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29f72991d2ed8e1d48f9d182db8c3af24c4afcc6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190524115966Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42536631296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125082966528, + "active_bytes_peak_delta": 14182873600, + "after": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 125082966528, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 125082966528, + "device": 5, + "inactive_split_bytes_current": 16131195904, + "inactive_split_bytes_peak": 28286038016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 129580924928, + "reserved_bytes_peak": 129580924928, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:05:24.115535Z", + "world_size": 8 + }, + "allocated_bytes_current": 42536631296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125082966528, + "allocated_bytes_peak_delta": 14182873600, + "before": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 110900092928, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 110900092928, + "device": 5, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_peak": 28286038016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115043467264, + "reserved_bytes_peak": 115043467264, + "segment_count_current": 909, + "segment_count_peak": 909, + "timestamp_utc": "2026-08-26T19:05:23.261277Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16131195904, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28286038016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 129580924928, + "reserved_bytes_current_delta": 14537457664, + "reserved_bytes_peak": 129580924928, + "reserved_bytes_peak_delta": 14537457664, + "segment_count_current": 918, + "segment_count_current_delta": 9, + "segment_count_peak": 918, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190524115966Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.193701Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190524115966Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190524115966Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..952013288d40bf32aaf3399b7e579c59423a6d0d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190524115966Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e971d6651616ed11413034ac4af93a542211e0e48841cba783f1169468f4db2 +size 4450242 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190525012087Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190525012087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf20160f40ef8b746d921504d80be6e9a53320f3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190525012087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42536631296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125100899328, + "active_bytes_peak_delta": 17932800, + "after": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 125100899328, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 125100899328, + "device": 5, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_peak": 28286038016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 129580924928, + "reserved_bytes_peak": 129580924928, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:05:25.011675Z", + "world_size": 8 + }, + "allocated_bytes_current": 42536631296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125100899328, + "allocated_bytes_peak_delta": 17932800, + "before": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 125082966528, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 125082966528, + "device": 5, + "inactive_split_bytes_current": 16131195904, + "inactive_split_bytes_peak": 28286038016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 129580924928, + "reserved_bytes_peak": 129580924928, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:05:24.203748Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28286038016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 129580924928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129580924928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 918, + "segment_count_current_delta": 0, + "segment_count_peak": 918, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190525012087Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.082049Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190525012087Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190525012087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..629875d5c5443756ea0d703696341ba1f0fe3151 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190525012087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0e83b5d9c36b33504e6bc508b06ccb050857307b2d130ae944e2d38a394c473 +size 4468025 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190525902018Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190525902018Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e9fcb6682f35b3a8f6549be33966856fd3d91ac --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190525902018Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42536631296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125100899328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 125100899328, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 125100899328, + "device": 5, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_peak": 28286038016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 129580924928, + "reserved_bytes_peak": 129580924928, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:05:25.901612Z", + "world_size": 8 + }, + "allocated_bytes_current": 42536631296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125100899328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 125100899328, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 125100899328, + "device": 5, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_peak": 28286038016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 129580924928, + "reserved_bytes_peak": 129580924928, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:05:25.090534Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28286038016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 129580924928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129580924928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 918, + "segment_count_current_delta": 0, + "segment_count_peak": 918, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190525902018Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.972480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190525902018Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190525902018Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea24677d6a56378041fb30177473257718b190fd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190525902018Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e38a63b2bb4727742c7a31c5e88c970a6895e8202868d92cb04ea9ffd8a10a02 +size 4467972 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190526795514Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190526795514Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68838513928aea0596a346e478e09a4a8d1e7101 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190526795514Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42536631296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125100899328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 125100899328, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 125100899328, + "device": 5, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_peak": 28286038016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 129580924928, + "reserved_bytes_peak": 129580924928, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:05:26.795102Z", + "world_size": 8 + }, + "allocated_bytes_current": 42536631296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125100899328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42536631296, + "active_bytes_peak": 125100899328, + "allocated_bytes_current": 42536631296, + "allocated_bytes_peak": 125100899328, + "device": 5, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_peak": 28286038016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 129580924928, + "reserved_bytes_peak": 129580924928, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-26T19:05:25.980598Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16223470592, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28286038016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 129580924928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129580924928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 918, + "segment_count_current_delta": 0, + "segment_count_peak": 918, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190526795514Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.864056Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190526795514Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190526795514Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc360cb8bbd25bae1616580502f7129058d35237 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190526795514Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff0481b24cd422b33aad5de55d77b7cf7103d224f9d352e0b89225f848174094 +size 4468016 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190523170368Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190523170368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4165b31b85a852dafb8d4adc5f391c69b4599383 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190523170368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534990336, + "active_bytes_current_delta": 19489243136, + "active_bytes_peak": 110902932480, + "active_bytes_peak_delta": 87844602368, + "after": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 110902932480, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 110902932480, + "device": 6, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_peak": 28274731008, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115183976448, + "reserved_bytes_peak": 115183976448, + "segment_count_current": 916, + "segment_count_peak": 916, + "timestamp_utc": "2026-08-26T19:05:23.169877Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534990336, + "allocated_bytes_current_delta": 19489243136, + "allocated_bytes_peak": 110902932480, + "allocated_bytes_peak_delta": 87844602368, + "before": { + "active_bytes_current": 23045747200, + "active_bytes_peak": 23058330112, + "allocated_bytes_current": 23045747200, + "allocated_bytes_peak": 23058330112, + "device": 6, + "inactive_split_bytes_current": 358469120, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25761415168, + "reserved_bytes_peak": 25761415168, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-26T19:04:39.309072Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_current_delta": 16040706048, + "inactive_split_bytes_peak": 28274731008, + "inactive_split_bytes_peak_delta": 27391495680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115183976448, + "reserved_bytes_current_delta": 89422561280, + "reserved_bytes_peak": 115183976448, + "reserved_bytes_peak_delta": 89422561280, + "segment_count_current": 916, + "segment_count_current_delta": 359, + "segment_count_peak": 916, + "segment_count_peak_delta": 359, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190523170368Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.251668Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190523170368Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190523170368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e647e0ea0734a3355b9563aab787ace59b90d6d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190523170368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7085b2e7b958ea3bd96084de4a366dcad7af2c2732add8d2278ab9b69b3aef44 +size 4081080 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190524116000Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190524116000Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1458670ed59d73f728958aeebc137edb10c69519 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190524116000Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534990336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125111505408, + "active_bytes_peak_delta": 14208572928, + "after": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 125111505408, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 125111505408, + "device": 6, + "inactive_split_bytes_current": 16306900480, + "inactive_split_bytes_peak": 28274731008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 129727725568, + "reserved_bytes_peak": 129727725568, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:05:24.115593Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534990336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125111505408, + "allocated_bytes_peak_delta": 14208572928, + "before": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 110902932480, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 110902932480, + "device": 6, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_peak": 28274731008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115183976448, + "reserved_bytes_peak": 115183976448, + "segment_count_current": 916, + "segment_count_peak": 916, + "timestamp_utc": "2026-08-26T19:05:23.262238Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16306900480, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28274731008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 129727725568, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 129727725568, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 928, + "segment_count_current_delta": 12, + "segment_count_peak": 928, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190524116000Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.192897Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190524116000Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190524116000Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bbb1624ef20a5cd07cc11ee210f125df1ed4e098 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190524116000Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0dbbc6b11dd0f04e40db2de5e724db31e98c9d97057a65bb3431d03f9df5848 +size 4481219 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190525012126Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190525012126Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b430928d0ebe9e9146ac734a5fa1e6ad5dc16a3d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190525012126Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534990336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125128464896, + "active_bytes_peak_delta": 16959488, + "after": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 125128464896, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 125128464896, + "device": 6, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_peak": 28274731008, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 129727725568, + "reserved_bytes_peak": 129727725568, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:05:25.011681Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534990336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125128464896, + "allocated_bytes_peak_delta": 16959488, + "before": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 125111505408, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 125111505408, + "device": 6, + "inactive_split_bytes_current": 16306900480, + "inactive_split_bytes_peak": 28274731008, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 129727725568, + "reserved_bytes_peak": 129727725568, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:05:24.200848Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28274731008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 129727725568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129727725568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190525012126Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.087026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190525012126Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190525012126Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b07d65cd5aabfb72c26b41f2fc76ae050e149d5c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190525012126Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94acee2fd7f486f6026372692a2371df59f59b635f89e86f5d8924d6926d7e24 +size 4499026 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190525902072Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190525902072Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f9a567a363ec52da115714240193352734be98d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190525902072Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534990336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125128464896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 125128464896, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 125128464896, + "device": 6, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_peak": 28274731008, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 129727725568, + "reserved_bytes_peak": 129727725568, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:05:25.901650Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534990336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125128464896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 125128464896, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 125128464896, + "device": 6, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_peak": 28274731008, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 129727725568, + "reserved_bytes_peak": 129727725568, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:05:25.095374Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28274731008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 129727725568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129727725568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190525902072Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.970159Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190525902072Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190525902072Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a81835aeb48ff817d52043fe03e23e5c28f09cfd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190525902072Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0b93f26dfc6f206d9fbbe9292e35249fced50068ac1181ab8db3967e1fc3f9 +size 4498955 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190526795533Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190526795533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ceafe200e2f942fb658af9287c3667f798642c80 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190526795533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42534990336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125133843968, + "active_bytes_peak_delta": 5379072, + "after": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 125133843968, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 125133843968, + "device": 6, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_peak": 28274731008, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 129727725568, + "reserved_bytes_peak": 129727725568, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:05:26.795123Z", + "world_size": 8 + }, + "allocated_bytes_current": 42534990336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125133843968, + "allocated_bytes_peak_delta": 5379072, + "before": { + "active_bytes_current": 42534990336, + "active_bytes_peak": 125128464896, + "allocated_bytes_current": 42534990336, + "allocated_bytes_peak": 125128464896, + "device": 6, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_peak": 28274731008, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 129727725568, + "reserved_bytes_peak": 129727725568, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-26T19:05:25.978098Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16399175168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28274731008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 129727725568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129727725568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190526795533Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.862588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190526795533Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190526795533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f763e05a7f196e8a17c9d1046b17e93c5cd13c7f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190526795533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:968fd66101863bede24502cae07afdb4a2ec13e270a6bae70689427f4fa65db3 +size 4499033 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190523170368Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190523170368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5bbc7de974c8d78d6495dbb78ce4d07ea261d0c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190523170368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42530020864, + "active_bytes_current_delta": 19481715200, + "active_bytes_peak": 110913328128, + "active_bytes_peak_delta": 87852439552, + "after": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 110913328128, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 110913328128, + "device": 7, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_peak": 28319159808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115204947968, + "reserved_bytes_peak": 115204947968, + "segment_count_current": 916, + "segment_count_peak": 916, + "timestamp_utc": "2026-08-26T19:05:23.169904Z", + "world_size": 8 + }, + "allocated_bytes_current": 42530020864, + "allocated_bytes_current_delta": 19481715200, + "allocated_bytes_peak": 110913328128, + "allocated_bytes_peak_delta": 87852439552, + "before": { + "active_bytes_current": 23048305664, + "active_bytes_peak": 23060888576, + "allocated_bytes_current": 23048305664, + "allocated_bytes_peak": 23060888576, + "device": 7, + "inactive_split_bytes_current": 502711296, + "inactive_split_bytes_peak": 883235328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:04:39.309081Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_current_delta": 15888850432, + "inactive_split_bytes_peak": 28319159808, + "inactive_split_bytes_peak_delta": 27435924480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115204947968, + "reserved_bytes_current_delta": 89420464128, + "reserved_bytes_peak": 115204947968, + "reserved_bytes_peak_delta": 89420464128, + "segment_count_current": 916, + "segment_count_current_delta": 358, + "segment_count_peak": 916, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190523170368Z.pickle", + "timestamp_utc": "2026-08-26T19:05:23.250101Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190523170368Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190523170368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d2546b9d237c691d5f15226c4edd92437f34fda --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190523170368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1250a7e921e2750b567fc5c774ff0ccd23bc65d1089751b6c8d1d3e088c7e799 +size 4044664 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190524116006Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190524116006Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59cbaff5d54be0c03fd26403bee5d163c54da8bf --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190524116006Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42530020864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125092402176, + "active_bytes_peak_delta": 14179074048, + "after": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 125092402176, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 125092402176, + "device": 7, + "inactive_split_bytes_current": 16299287040, + "inactive_split_bytes_peak": 28319159808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 129742405632, + "reserved_bytes_peak": 129742405632, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:24.115581Z", + "world_size": 8 + }, + "allocated_bytes_current": 42530020864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125092402176, + "allocated_bytes_peak_delta": 14179074048, + "before": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 110913328128, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 110913328128, + "device": 7, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_peak": 28319159808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115204947968, + "reserved_bytes_peak": 115204947968, + "segment_count_current": 916, + "segment_count_peak": 916, + "timestamp_utc": "2026-08-26T19:05:23.261067Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16299287040, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 28319159808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 129742405632, + "reserved_bytes_current_delta": 14537457664, + "reserved_bytes_peak": 129742405632, + "reserved_bytes_peak_delta": 14537457664, + "segment_count_current": 925, + "segment_count_current_delta": 9, + "segment_count_peak": 925, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190524116006Z.pickle", + "timestamp_utc": "2026-08-26T19:05:24.194062Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190524116006Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190524116006Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8e7f65250761de481ccc1754984d6ce534e7f212 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190524116006Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcc5af79bf3f8f5afde05dda66fa05ecf134520f58c1769afb277f12a427ccc6 +size 4449490 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190525012133Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190525012133Z.json new file mode 100644 index 0000000000000000000000000000000000000000..51dd02db5970cda667b42230989be621e04792fc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190525012133Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42530020864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125092402176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 125092402176, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 125092402176, + "device": 7, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_peak": 28319159808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 129742405632, + "reserved_bytes_peak": 129742405632, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.011702Z", + "world_size": 8 + }, + "allocated_bytes_current": 42530020864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125092402176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 125092402176, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 125092402176, + "device": 7, + "inactive_split_bytes_current": 16299287040, + "inactive_split_bytes_peak": 28319159808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 129742405632, + "reserved_bytes_peak": 129742405632, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:24.202802Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 28319159808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 129742405632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129742405632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 925, + "segment_count_current_delta": 0, + "segment_count_peak": 925, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190525012133Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.085952Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190525012133Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190525012133Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..402f7754aeb1d7e35d5b38f2dc1590d13fd1f506 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190525012133Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23c4f0ae767df5c899ac640679719cc1a06d8de9c0ef0722201421b0b57d3bde +size 4467500 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190525902136Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190525902136Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec345b38f42a293ae8925367a31658fd82118a8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190525902136Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42530020864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125092402176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 125092402176, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 125092402176, + "device": 7, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_peak": 28319159808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 129742405632, + "reserved_bytes_peak": 129742405632, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.901724Z", + "world_size": 8 + }, + "allocated_bytes_current": 42530020864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125092402176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 125092402176, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 125092402176, + "device": 7, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_peak": 28319159808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 129742405632, + "reserved_bytes_peak": 129742405632, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.094598Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28319159808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 129742405632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129742405632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 925, + "segment_count_current_delta": 0, + "segment_count_peak": 925, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190525902136Z.pickle", + "timestamp_utc": "2026-08-26T19:05:25.973562Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190525902136Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190525902136Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e91042777ddd24d36c5e571217d150ddd557752 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190525902136Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d97648a05063c3da315c4b7fb8d854f3a63448d454c294a89e826d2b47bad49 +size 4467462 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190526795587Z.json b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190526795587Z.json new file mode 100644 index 0000000000000000000000000000000000000000..20a49f642c3a084f4c046447972ee7b6784e9bdb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190526795587Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 42530020864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 125092402176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 125092402176, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 125092402176, + "device": 7, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_peak": 28319159808, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 129742405632, + "reserved_bytes_peak": 129742405632, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:26.795177Z", + "world_size": 8 + }, + "allocated_bytes_current": 42530020864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 125092402176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 42530020864, + "active_bytes_peak": 125092402176, + "allocated_bytes_current": 42530020864, + "allocated_bytes_peak": 125092402176, + "device": 7, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_peak": 28319159808, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 129742405632, + "reserved_bytes_peak": 129742405632, + "segment_count_current": 925, + "segment_count_peak": 925, + "timestamp_utc": "2026-08-26T19:05:25.982133Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16391561728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28319159808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 129742405632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 129742405632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 925, + "segment_count_current_delta": 0, + "segment_count_peak": 925, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190526795587Z.pickle", + "timestamp_utc": "2026-08-26T19:05:26.865296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190526795587Z.pickle b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190526795587Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e88986494df9260ff5713b8e4dbd27abe84d5be --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190526795587Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78c2a05ee39fbed7ab21a9f1d943fd76cb583bca9a91bd945761fc3bb0277b58 +size 4467464 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..c17c6a860c093a47bfa975c341da2907f0fe0438 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=0 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=dynamic +capacity=dynamic +cuda_graph=0 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5d6d1258fd1af50972944c065c8b88f395a54cee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log @@ -0,0 +1,1633 @@ +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=dynamic cap=dynamic cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 19:04:14.803000 605565 torch/distributed/run.py:766] +W0826 19:04:14.803000 605565 torch/distributed/run.py:766] ***************************************** +W0826 19:04:14.803000 605565 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:04:14.803000 605565 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... False + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 74 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 19:04:32.695167814 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:04:32.697473201 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:04:32.697514757 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:04:32.697578863 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:04:32.697599190 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:04:32.699277153 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:04:32.699294310 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:04:32.862697851 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.730 seconds +time to initialize megatron (seconds): 7.169 +[after megatron is initialized] datetime: 2026-08-26 19:04:38 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.input_layernorm.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:04:39 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:04:39 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (619.28, 631.70) + train/valid/test-data-iterators-setup ..........: (62.30, 68.04) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:04:39 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:05:23] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 43857.7 | throughput per GPU (TFLOP/s/GPU): 2.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.621 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 40563.6708984375 | max allocated: 105778.8759765625 | reserved: 109782.0 | max reserved: 109782.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:05:24] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 949.8 | throughput per GPU (TFLOP/s/GPU): 134.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:05:25] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 896.1 | throughput per GPU (TFLOP/s/GPU): 142.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.953032E+00 | loss scale: 1.0 | grad norm: 42.255 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:05:25] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 889.9 | throughput per GPU (TFLOP/s/GPU): 143.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477413E+00 | loss scale: 1.0 | grad norm: 7.147 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:05:26] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 893.4 | throughput per GPU (TFLOP/s/GPU): 142.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.116790E+01 | loss scale: 1.0 | grad norm: 399.552 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:05:26 +[rank0]:[W826 19:05:28.186328982 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..7ca702a66c14e1642cfc07062528f7f56d4dc70e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 45 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 1079 +1, 1079 +2, 1079 +3, 1061 +4, 1077 +5, 1079 +6, 1079 +7, 1069 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 43759 +1, 43215 +2, 43187 +3, 43877 +4, 43885 +5, 43895 +6, 43905 +7, 43913 +0, 44193 +1, 43625 +2, 43645 +3, 44345 +4, 44345 +5, 44365 +6, 44365 +7, 44375 +0, 47239 +1, 46671 +2, 46671 +3, 47371 +4, 47371 +5, 47371 +6, 47371 +7, 47371 +0, 47519 +1, 46951 +2, 46951 +3, 47651 +4, 47651 +5, 47651 +6, 47651 +7, 47651 +0, 47759 +1, 47183 +2, 47187 +3, 47877 +4, 47865 +5, 47879 +6, 47903 +7, 47887 +0, 48553 +1, 47985 +2, 47985 +3, 48685 +4, 48685 +5, 48685 +6, 48685 +7, 48685 +0, 48777 +1, 48209 +2, 48209 +3, 48909 +4, 48909 +5, 48909 +6, 48909 +7, 48909 +0, 49341 +1, 48771 +2, 48771 +3, 49471 +4, 49471 +5, 49471 +6, 49471 +7, 49471 +0, 49859 +1, 49291 +2, 49291 +3, 49991 +4, 49991 +5, 49991 +6, 49991 +7, 49991 +0, 77071 +1, 77021 +2, 77021 +3, 77055 +4, 77055 +5, 77055 +6, 77055 +7, 77055 +0, 77717 +1, 77667 +2, 77667 +3, 77701 +4, 77701 +5, 77701 +6, 77701 +7, 77701 +0, 77717 +1, 77667 +2, 77667 +3, 77701 +4, 77701 +5, 77701 +6, 77701 +7, 77701 +0, 78037 +1, 77987 +2, 77987 +3, 78021 +4, 78021 +5, 78021 +6, 78021 +7, 78021 +0, 78037 +1, 77987 +2, 77987 +3, 78021 +4, 78021 +5, 78021 +6, 78021 +7, 78021 +0, 78443 +1, 78419 +2, 78433 +3, 78447 +4, 78449 +5, 78407 +6, 78445 +7, 78415 +0, 81199 +1, 80869 +2, 81149 +3, 80903 +4, 80903 +5, 81183 +6, 80903 +7, 80903 +0, 89857 +1, 89373 +2, 89683 +3, 89463 +4, 89433 +5, 89699 +6, 89505 +7, 89477 +0, 92831 +1, 93265 +2, 98173 +3, 92787 +4, 94131 +5, 92133 +6, 94075 +7, 94245 +0, 111885 +1, 111547 +2, 111525 +3, 111901 +4, 111901 +5, 111901 +6, 111901 +7, 111901 +0, 2382 +1, 2382 +2, 2382 +3, 2638 +4, 111007 +5, 3662 +6, 2382 +7, 2382 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190638852000Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190638852000Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e1ba0f5b0a7dd8ae29b527e45f70d3165eb1e2e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190638852000Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084302336, + "active_bytes_current_delta": 31399380480, + "active_bytes_peak": 73298068480, + "active_bytes_peak_delta": 32575397888, + "after": { + "active_bytes_current": 72084302336, + "active_bytes_peak": 73298068480, + "allocated_bytes_current": 72084302336, + "allocated_bytes_peak": 73298068480, + "device": 0, + "inactive_split_bytes_current": 2456868352, + "inactive_split_bytes_peak": 4233220608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 85226160128, + "reserved_bytes_peak": 85226160128, + "segment_count_current": 1076, + "segment_count_peak": 1076, + "timestamp_utc": "2026-08-26T19:06:38.851534Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084302336, + "allocated_bytes_current_delta": 31399380480, + "allocated_bytes_peak": 73298068480, + "allocated_bytes_peak_delta": 32575397888, + "before": { + "active_bytes_current": 40684921856, + "active_bytes_peak": 40722670592, + "allocated_bytes_current": 40684921856, + "allocated_bytes_peak": 40722670592, + "device": 0, + "inactive_split_bytes_current": 373120000, + "inactive_split_bytes_peak": 593320960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 43723522048, + "reserved_bytes_peak": 43723522048, + "segment_count_current": 262, + "segment_count_peak": 262, + "timestamp_utc": "2026-08-26T19:05:59.393830Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2456868352, + "inactive_split_bytes_current_delta": 2083748352, + "inactive_split_bytes_peak": 4233220608, + "inactive_split_bytes_peak_delta": 3639899648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 85226160128, + "reserved_bytes_current_delta": 41502638080, + "reserved_bytes_peak": 85226160128, + "reserved_bytes_peak_delta": 41502638080, + "segment_count_current": 1076, + "segment_count_current_delta": 814, + "segment_count_peak": 1076, + "segment_count_peak_delta": 814, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190638852000Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.888917Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190638852000Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190638852000Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2cf0197df340ca310a57624184559900cfb9292a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190638852000Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1b9eaeb63cc16becb317f4771f626e40960441fbb7e5e572302e2ea430a756 +size 2068727 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190639459375Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190639459375Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce629fc7614f1a715d76a2b318c59118682cdd1c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190639459375Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084302336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94387964928, + "active_bytes_peak_delta": 21089896448, + "after": { + "active_bytes_current": 72084302336, + "active_bytes_peak": 94387964928, + "allocated_bytes_current": 72084302336, + "allocated_bytes_peak": 94387964928, + "device": 0, + "inactive_split_bytes_current": 2752566784, + "inactive_split_bytes_peak": 4607880704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 99950264320, + "reserved_bytes_peak": 99950264320, + "segment_count_current": 1151, + "segment_count_peak": 1151, + "timestamp_utc": "2026-08-26T19:06:39.458931Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084302336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94387964928, + "allocated_bytes_peak_delta": 21089896448, + "before": { + "active_bytes_current": 72084302336, + "active_bytes_peak": 73298068480, + "allocated_bytes_current": 72084302336, + "allocated_bytes_peak": 73298068480, + "device": 0, + "inactive_split_bytes_current": 2456868352, + "inactive_split_bytes_peak": 4233220608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 85226160128, + "reserved_bytes_peak": 85226160128, + "segment_count_current": 1076, + "segment_count_peak": 1076, + "timestamp_utc": "2026-08-26T19:06:38.893261Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2752566784, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4607880704, + "inactive_split_bytes_peak_delta": 374660096, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 99950264320, + "reserved_bytes_current_delta": 14724104192, + "reserved_bytes_peak": 99950264320, + "reserved_bytes_peak_delta": 14724104192, + "segment_count_current": 1151, + "segment_count_current_delta": 75, + "segment_count_peak": 1151, + "segment_count_peak_delta": 75, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190639459375Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.492652Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190639459375Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190639459375Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78a2c35706622d522e0c272c67a33dc24a044d93 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190639459375Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecd1ba1dfefcda062058fb82ceca4e35a5d8860806aad2079a37b6d1fef851de +size 1708179 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190639737394Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190639737394Z.json new file mode 100644 index 0000000000000000000000000000000000000000..228ec0acf713aff81c7cca0595d665e4f7952232 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190639737394Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084302336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94387964928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084302336, + "active_bytes_peak": 94387964928, + "allocated_bytes_current": 72084302336, + "allocated_bytes_peak": 94387964928, + "device": 0, + "inactive_split_bytes_current": 2752566784, + "inactive_split_bytes_peak": 4607880704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 99950264320, + "reserved_bytes_peak": 99950264320, + "segment_count_current": 1151, + "segment_count_peak": 1151, + "timestamp_utc": "2026-08-26T19:06:39.736981Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084302336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94387964928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084302336, + "active_bytes_peak": 94387964928, + "allocated_bytes_current": 72084302336, + "allocated_bytes_peak": 94387964928, + "device": 0, + "inactive_split_bytes_current": 2752566784, + "inactive_split_bytes_peak": 4607880704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 99950264320, + "reserved_bytes_peak": 99950264320, + "segment_count_current": 1151, + "segment_count_peak": 1151, + "timestamp_utc": "2026-08-26T19:06:39.496545Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2752566784, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4607880704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 99950264320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99950264320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1151, + "segment_count_current_delta": 0, + "segment_count_peak": 1151, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190639737394Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.765993Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190639737394Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190639737394Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d28c165c4d111ea88b07649515f5badf757e683c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190639737394Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3f6affd482cabc3783fbeb5768d7231657b3fa3806098402433c55b6700e1cd +size 1700428 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190641564681Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190641564681Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5855c310d134b393dbd5e4fe92d2536c49de6f9d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190641564681Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101345792, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98795091968, + "active_bytes_peak_delta": 4407127040, + "after": { + "active_bytes_current": 72101345792, + "active_bytes_peak": 98795091968, + "allocated_bytes_current": 72101345792, + "allocated_bytes_peak": 94391072768, + "device": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_peak": 7100247552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_peak": 108158517248, + "segment_count_current": 1223, + "segment_count_peak": 1223, + "timestamp_utc": "2026-08-26T19:06:41.564241Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101345792, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94391072768, + "allocated_bytes_peak_delta": 3107840, + "before": { + "active_bytes_current": 72084302336, + "active_bytes_peak": 94387964928, + "allocated_bytes_current": 72084302336, + "allocated_bytes_peak": 94387964928, + "device": 0, + "inactive_split_bytes_current": 2752566784, + "inactive_split_bytes_peak": 4607880704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 99950264320, + "reserved_bytes_peak": 99950264320, + "segment_count_current": 1151, + "segment_count_peak": 1151, + "timestamp_utc": "2026-08-26T19:06:39.769511Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_current_delta": -140775424, + "inactive_split_bytes_peak": 7100247552, + "inactive_split_bytes_peak_delta": 2492366848, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_current_delta": 8208252928, + "reserved_bytes_peak": 108158517248, + "reserved_bytes_peak_delta": 8208252928, + "segment_count_current": 1223, + "segment_count_current_delta": 72, + "segment_count_peak": 1223, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190641564681Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.600477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190641564681Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190641564681Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc802fad7385d21af520ae5369a4d96e6cd6877c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190641564681Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:014a143fd7d3cd156a531e8667ceebbefa93a994c789d7f7aa40f0c55caea219 +size 2176865 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190641791310Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190641791310Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f98fd9b7092077e4494dade48686099e50951e41 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190641791310Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101345792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795091968, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101345792, + "active_bytes_peak": 98795091968, + "allocated_bytes_current": 72101345792, + "allocated_bytes_peak": 94391072768, + "device": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_peak": 7100247552, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_peak": 108158517248, + "segment_count_current": 1223, + "segment_count_peak": 1223, + "timestamp_utc": "2026-08-26T19:06:41.790891Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101345792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391072768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101345792, + "active_bytes_peak": 98795091968, + "allocated_bytes_current": 72101345792, + "allocated_bytes_peak": 94391072768, + "device": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_peak": 7100247552, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_peak": 108158517248, + "segment_count_current": 1223, + "segment_count_peak": 1223, + "timestamp_utc": "2026-08-26T19:06:41.604495Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7100247552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108158517248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1223, + "segment_count_current_delta": 0, + "segment_count_peak": 1223, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190641791310Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798607Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190641791310Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190641791310Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d803424ca3bfd0adefcf9e2a7daf5f1f4cfbe88 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190641791310Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46e05dcb0d3e17cd59f4d529eefd92ff8c051093e7089139e8e51e5f16b60666 +size 355715 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190641979691Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190641979691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32ca3d113c91efa7ae424539d1113c8fe933ed2a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190641979691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101345792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795091968, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101345792, + "active_bytes_peak": 98795091968, + "allocated_bytes_current": 72101345792, + "allocated_bytes_peak": 94391072768, + "device": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_peak": 7100247552, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_peak": 108158517248, + "segment_count_current": 1223, + "segment_count_peak": 1223, + "timestamp_utc": "2026-08-26T19:06:41.979311Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101345792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391072768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101345792, + "active_bytes_peak": 98795091968, + "allocated_bytes_current": 72101345792, + "allocated_bytes_peak": 94391072768, + "device": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_peak": 7100247552, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_peak": 108158517248, + "segment_count_current": 1223, + "segment_count_peak": 1223, + "timestamp_utc": "2026-08-26T19:06:41.799639Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2611791360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7100247552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108158517248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108158517248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1223, + "segment_count_current_delta": 0, + "segment_count_peak": 1223, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T190641979691Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986820Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190641979691Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190641979691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c5b6a565726430c38416ef1e23a9090779dbf06 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190641979691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d7236615cc500cb2cfd796ad1eac2d0d59c42c1f5aa88e3a822719de3f7f608 +size 355715 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190638851999Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190638851999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d81af087f9379658856e867b530559ad9838ed4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190638851999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72065014272, + "active_bytes_current_delta": 31379045888, + "active_bytes_peak": 73306037248, + "active_bytes_peak_delta": 32582320128, + "after": { + "active_bytes_current": 72065014272, + "active_bytes_peak": 73306037248, + "allocated_bytes_current": 72065014272, + "allocated_bytes_peak": 73306037248, + "device": 1, + "inactive_split_bytes_current": 2121737728, + "inactive_split_bytes_peak": 4097391616, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 84729135104, + "reserved_bytes_peak": 84729135104, + "segment_count_current": 1077, + "segment_count_peak": 1077, + "timestamp_utc": "2026-08-26T19:06:38.851496Z", + "world_size": 8 + }, + "allocated_bytes_current": 72065014272, + "allocated_bytes_current_delta": 31379045888, + "allocated_bytes_peak": 73306037248, + "allocated_bytes_peak_delta": 32582320128, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 1, + "inactive_split_bytes_current": 302867456, + "inactive_split_bytes_peak": 523068416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:05:59.393796Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2121737728, + "inactive_split_bytes_current_delta": 1818870272, + "inactive_split_bytes_peak": 4097391616, + "inactive_split_bytes_peak_delta": 3574323200, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 84729135104, + "reserved_bytes_current_delta": 41599107072, + "reserved_bytes_peak": 84729135104, + "reserved_bytes_peak_delta": 41599107072, + "segment_count_current": 1077, + "segment_count_current_delta": 826, + "segment_count_peak": 1077, + "segment_count_peak_delta": 826, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190638851999Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.888125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190638851999Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190638851999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..705bd95ee1227a35b8cbe312aa98796fcefac61e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190638851999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92f8ab429e67eb0b1403f36f7fa1c6fe1442e0d2f901b54af9b234c47797815d +size 1804787 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190639459363Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190639459363Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e4b1827357a34434272473fb9ba180dec47e5b8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190639459363Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72065014272, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94387663360, + "active_bytes_peak_delta": 21081626112, + "after": { + "active_bytes_current": 72065014272, + "active_bytes_peak": 94387663360, + "allocated_bytes_current": 72065014272, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2415339008, + "inactive_split_bytes_peak": 4486276608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 99713286144, + "reserved_bytes_peak": 99713286144, + "segment_count_current": 1173, + "segment_count_peak": 1173, + "timestamp_utc": "2026-08-26T19:06:39.458945Z", + "world_size": 8 + }, + "allocated_bytes_current": 72065014272, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94387663360, + "allocated_bytes_peak_delta": 21081626112, + "before": { + "active_bytes_current": 72065014272, + "active_bytes_peak": 73306037248, + "allocated_bytes_current": 72065014272, + "allocated_bytes_peak": 73306037248, + "device": 1, + "inactive_split_bytes_current": 2121737728, + "inactive_split_bytes_peak": 4097391616, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 84729135104, + "reserved_bytes_peak": 84729135104, + "segment_count_current": 1077, + "segment_count_peak": 1077, + "timestamp_utc": "2026-08-26T19:06:38.892386Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2415339008, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4486276608, + "inactive_split_bytes_peak_delta": 388884992, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 99713286144, + "reserved_bytes_current_delta": 14984151040, + "reserved_bytes_peak": 99713286144, + "reserved_bytes_peak_delta": 14984151040, + "segment_count_current": 1173, + "segment_count_current_delta": 96, + "segment_count_peak": 1173, + "segment_count_peak_delta": 96, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190639459363Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.492231Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190639459363Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190639459363Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0abdc3825a7aaa8c4402d96c980ae2dcb3c4bf22 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190639459363Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6439f46f7392168ec3fe84449d64167633da01b073a28a51e7a4bafc48dfd867 +size 1706488 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190639737395Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190639737395Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af2938d22cf743b48e518221eefdbd95f004dfb2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190639737395Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72065014272, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94387663360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72065014272, + "active_bytes_peak": 94387663360, + "allocated_bytes_current": 72065014272, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2417436160, + "inactive_split_bytes_peak": 4486276608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 99713286144, + "reserved_bytes_peak": 99713286144, + "segment_count_current": 1173, + "segment_count_peak": 1173, + "timestamp_utc": "2026-08-26T19:06:39.736977Z", + "world_size": 8 + }, + "allocated_bytes_current": 72065014272, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94387663360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72065014272, + "active_bytes_peak": 94387663360, + "allocated_bytes_current": 72065014272, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2415339008, + "inactive_split_bytes_peak": 4486276608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 99713286144, + "reserved_bytes_peak": 99713286144, + "segment_count_current": 1173, + "segment_count_peak": 1173, + "timestamp_utc": "2026-08-26T19:06:39.495974Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2417436160, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 4486276608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 99713286144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99713286144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1173, + "segment_count_current_delta": 0, + "segment_count_peak": 1173, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190639737395Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.765098Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190639737395Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190639737395Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..051dd80f18f5d835313fd30f438aea502ee6813d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190639737395Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81e6667db4315050607030d0cb6f47ea49c271193e4eca21a8b4bb25272ecaf1 +size 1715397 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190641564744Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190641564744Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e76d7343502e3481e7f2476c554713d7195c52c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190641564744Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72082057728, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98775803904, + "active_bytes_peak_delta": 4388140544, + "after": { + "active_bytes_current": 72082057728, + "active_bytes_peak": 98775803904, + "allocated_bytes_current": 72082057728, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_peak": 6530235904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107804098560, + "reserved_bytes_peak": 107804098560, + "segment_count_current": 1217, + "segment_count_peak": 1217, + "timestamp_utc": "2026-08-26T19:06:41.564292Z", + "world_size": 8 + }, + "allocated_bytes_current": 72082057728, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94387663360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72065014272, + "active_bytes_peak": 94387663360, + "allocated_bytes_current": 72065014272, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2417436160, + "inactive_split_bytes_peak": 4486276608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 99713286144, + "reserved_bytes_peak": 99713286144, + "segment_count_current": 1173, + "segment_count_peak": 1173, + "timestamp_utc": "2026-08-26T19:06:39.768427Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_current_delta": -142872576, + "inactive_split_bytes_peak": 6530235904, + "inactive_split_bytes_peak_delta": 2043959296, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107804098560, + "reserved_bytes_current_delta": 8090812416, + "reserved_bytes_peak": 107804098560, + "reserved_bytes_peak_delta": 8090812416, + "segment_count_current": 1217, + "segment_count_current_delta": 44, + "segment_count_peak": 1217, + "segment_count_peak_delta": 44, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190641564744Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.599252Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190641564744Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190641564744Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02b302a77d73c3b4429a14ee88e274473df3df65 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190641564744Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e25d641b709f9b1efadd731ba3895c2833ba1ec92608f3d486a003c25ae4be9a +size 2098871 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190641791300Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190641791300Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6d513ed22fd6cc496e86e9b1460266dc3d5b61f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190641791300Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72082057728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98775803904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72082057728, + "active_bytes_peak": 98775803904, + "allocated_bytes_current": 72082057728, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_peak": 6530235904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107804098560, + "reserved_bytes_peak": 107804098560, + "segment_count_current": 1217, + "segment_count_peak": 1217, + "timestamp_utc": "2026-08-26T19:06:41.790892Z", + "world_size": 8 + }, + "allocated_bytes_current": 72082057728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94387663360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72082057728, + "active_bytes_peak": 98775803904, + "allocated_bytes_current": 72082057728, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_peak": 6530235904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107804098560, + "reserved_bytes_peak": 107804098560, + "segment_count_current": 1217, + "segment_count_peak": 1217, + "timestamp_utc": "2026-08-26T19:06:41.603118Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6530235904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107804098560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107804098560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1217, + "segment_count_current_delta": 0, + "segment_count_peak": 1217, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190641791300Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190641791300Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190641791300Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..168e14ced58da228eda35aaa08efaa3db4dbac17 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190641791300Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d7fda7bd85b18746370b330214be44c643f70fa2e29c6aa2ae3d11771692d0c +size 337977 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190641979724Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190641979724Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f3c3d0fa7555526a0d153ace739d200902de1d6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190641979724Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72082057728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98775803904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72082057728, + "active_bytes_peak": 98775803904, + "allocated_bytes_current": 72082057728, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_peak": 6530235904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107804098560, + "reserved_bytes_peak": 107804098560, + "segment_count_current": 1217, + "segment_count_peak": 1217, + "timestamp_utc": "2026-08-26T19:06:41.979333Z", + "world_size": 8 + }, + "allocated_bytes_current": 72082057728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94387663360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72082057728, + "active_bytes_peak": 98775803904, + "allocated_bytes_current": 72082057728, + "allocated_bytes_peak": 94387663360, + "device": 1, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_peak": 6530235904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107804098560, + "reserved_bytes_peak": 107804098560, + "segment_count_current": 1217, + "segment_count_peak": 1217, + "timestamp_utc": "2026-08-26T19:06:41.799410Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2274563584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6530235904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107804098560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107804098560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1217, + "segment_count_current_delta": 0, + "segment_count_peak": 1217, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190641979724Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190641979724Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190641979724Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5606992d4abdf2f253658158ad7a283441a394a8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190641979724Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5567e67c1c06ba435fe21ab3b5c115fe60c3288a5611ad1c6a2e5f5b7421d573 +size 337977 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190638852022Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190638852022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa855bd98b360d42b5b91225fac77ee947a4870d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190638852022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72065211904, + "active_bytes_current_delta": 31379243520, + "active_bytes_peak": 73305741312, + "active_bytes_peak_delta": 32582024192, + "after": { + "active_bytes_current": 72065211904, + "active_bytes_peak": 73305741312, + "allocated_bytes_current": 72065211904, + "allocated_bytes_peak": 73305741312, + "device": 2, + "inactive_split_bytes_current": 2100568576, + "inactive_split_bytes_peak": 4077269504, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 85022736384, + "reserved_bytes_peak": 85022736384, + "segment_count_current": 1078, + "segment_count_peak": 1078, + "timestamp_utc": "2026-08-26T19:06:38.851527Z", + "world_size": 8 + }, + "allocated_bytes_current": 72065211904, + "allocated_bytes_current_delta": 31379243520, + "allocated_bytes_peak": 73305741312, + "allocated_bytes_peak_delta": 32582024192, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 2, + "inactive_split_bytes_current": 283993088, + "inactive_split_bytes_peak": 504194048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:05:59.393821Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2100568576, + "inactive_split_bytes_current_delta": 1816575488, + "inactive_split_bytes_peak": 4077269504, + "inactive_split_bytes_peak_delta": 3573075456, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 85022736384, + "reserved_bytes_current_delta": 41892708352, + "reserved_bytes_peak": 85022736384, + "reserved_bytes_peak_delta": 41892708352, + "segment_count_current": 1078, + "segment_count_current_delta": 827, + "segment_count_peak": 1078, + "segment_count_peak_delta": 827, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190638852022Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.888561Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190638852022Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190638852022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e62d25e8c643b57d6717a441915851caa2dcbbf2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190638852022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c88654a194eb71f1f21d35378712e743f39f5dcfd572e03487d9820b51ff36b7 +size 1802074 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190639459464Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190639459464Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1575ab96ed32d3cac495315b520599a68bb3e2a9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190639459464Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72065211904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94395322368, + "active_bytes_peak_delta": 21089581056, + "after": { + "active_bytes_current": 72065211904, + "active_bytes_peak": 94395322368, + "allocated_bytes_current": 72065211904, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2396267008, + "inactive_split_bytes_peak": 4596562432, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 99983818752, + "reserved_bytes_peak": 99983818752, + "segment_count_current": 1178, + "segment_count_peak": 1178, + "timestamp_utc": "2026-08-26T19:06:39.459036Z", + "world_size": 8 + }, + "allocated_bytes_current": 72065211904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94395322368, + "allocated_bytes_peak_delta": 21089581056, + "before": { + "active_bytes_current": 72065211904, + "active_bytes_peak": 73305741312, + "allocated_bytes_current": 72065211904, + "allocated_bytes_peak": 73305741312, + "device": 2, + "inactive_split_bytes_current": 2100568576, + "inactive_split_bytes_peak": 4077269504, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 85022736384, + "reserved_bytes_peak": 85022736384, + "segment_count_current": 1078, + "segment_count_peak": 1078, + "timestamp_utc": "2026-08-26T19:06:38.892788Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2396267008, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4596562432, + "inactive_split_bytes_peak_delta": 519292928, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 99983818752, + "reserved_bytes_current_delta": 14961082368, + "reserved_bytes_peak": 99983818752, + "reserved_bytes_peak_delta": 14961082368, + "segment_count_current": 1178, + "segment_count_current_delta": 100, + "segment_count_peak": 1178, + "segment_count_peak_delta": 100, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190639459464Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.492895Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190639459464Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190639459464Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22cf8faeb396087361058d5d40ef3b18d247b3c1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190639459464Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08982a63df3734f80843c6b9815efd01825f1ddeca1c6dbfead3362b537769c7 +size 1705045 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190639737349Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190639737349Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6fd4c6bb3e18fd0cfa152ce26159f2094ac61bc5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190639737349Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72065211904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94395322368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72065211904, + "active_bytes_peak": 94395322368, + "allocated_bytes_current": 72065211904, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2394169856, + "inactive_split_bytes_peak": 4596562432, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 99983818752, + "reserved_bytes_peak": 99983818752, + "segment_count_current": 1178, + "segment_count_peak": 1178, + "timestamp_utc": "2026-08-26T19:06:39.736957Z", + "world_size": 8 + }, + "allocated_bytes_current": 72065211904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94395322368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72065211904, + "active_bytes_peak": 94395322368, + "allocated_bytes_current": 72065211904, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2396267008, + "inactive_split_bytes_peak": 4596562432, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 99983818752, + "reserved_bytes_peak": 99983818752, + "segment_count_current": 1178, + "segment_count_peak": 1178, + "timestamp_utc": "2026-08-26T19:06:39.496636Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2394169856, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4596562432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 99983818752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99983818752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1178, + "segment_count_current_delta": 0, + "segment_count_peak": 1178, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190639737349Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.765324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190639737349Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190639737349Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1368c4d236450bba32e78f08169a971374014347 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190639737349Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:875cfbbc1d4f45762c435e559cbe82d9f903203ee03262404fe4ef2dccbb1645 +size 1709838 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190641564688Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190641564688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96e349b152c0d346e384c587dddcd6d15cc28cce --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190641564688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72082255360, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98776001536, + "active_bytes_peak_delta": 4380679168, + "after": { + "active_bytes_current": 72082255360, + "active_bytes_peak": 98776001536, + "allocated_bytes_current": 72082255360, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_peak": 6743947776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107781029888, + "reserved_bytes_peak": 107781029888, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.564246Z", + "world_size": 8 + }, + "allocated_bytes_current": 72082255360, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94395322368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72065211904, + "active_bytes_peak": 94395322368, + "allocated_bytes_current": 72065211904, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2394169856, + "inactive_split_bytes_peak": 4596562432, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 99983818752, + "reserved_bytes_peak": 99983818752, + "segment_count_current": 1178, + "segment_count_peak": 1178, + "timestamp_utc": "2026-08-26T19:06:39.768727Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_current_delta": -138678272, + "inactive_split_bytes_peak": 6743947776, + "inactive_split_bytes_peak_delta": 2147385344, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107781029888, + "reserved_bytes_current_delta": 7797211136, + "reserved_bytes_peak": 107781029888, + "reserved_bytes_peak_delta": 7797211136, + "segment_count_current": 1214, + "segment_count_current_delta": 36, + "segment_count_peak": 1214, + "segment_count_peak_delta": 36, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190641564688Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.599517Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190641564688Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190641564688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d75d00a562cba6d8d23952ed7ee34b74a922152b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190641564688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffbc57ff2e39bc90ed735c7b28dac71e93fd4dad403a311aaeeb3f9ff4d1d326 +size 2096524 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190641791261Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190641791261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b53afdc94790f4834622ee45324df3082fa9d28 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190641791261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72082255360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98776001536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72082255360, + "active_bytes_peak": 98776001536, + "allocated_bytes_current": 72082255360, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_peak": 6743947776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107781029888, + "reserved_bytes_peak": 107781029888, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.790869Z", + "world_size": 8 + }, + "allocated_bytes_current": 72082255360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94395322368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72082255360, + "active_bytes_peak": 98776001536, + "allocated_bytes_current": 72082255360, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_peak": 6743947776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107781029888, + "reserved_bytes_peak": 107781029888, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.603392Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6743947776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107781029888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107781029888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190641791261Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798304Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190641791261Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190641791261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cff0afe6a450c806a35ebde5f7d44a25afef1ed0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190641791261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db89915f931dd9554fdd623d35f4dcafc3957ba85cf5c30ae24d99023f019883 +size 335845 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190641979699Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190641979699Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d369efc678b5a4122a06f86aa056376f901251b8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190641979699Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72082255360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98776001536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72082255360, + "active_bytes_peak": 98776001536, + "allocated_bytes_current": 72082255360, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_peak": 6743947776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107781029888, + "reserved_bytes_peak": 107781029888, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.979324Z", + "world_size": 8 + }, + "allocated_bytes_current": 72082255360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94395322368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72082255360, + "active_bytes_peak": 98776001536, + "allocated_bytes_current": 72082255360, + "allocated_bytes_peak": 94395322368, + "device": 2, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_peak": 6743947776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107781029888, + "reserved_bytes_peak": 107781029888, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.799336Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2255491584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6743947776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107781029888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107781029888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190641979699Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986667Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190641979699Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190641979699Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ae856ada907a5060b84372c6e4a57487d18848b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190641979699Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7433d3ae7e7d13de89b374d261806725c1da452e2f728dbb200955c8b2c944b +size 335876 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190638852254Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190638852254Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a5026bb54d1d6307572e64071260257538d1849 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190638852254Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 31398314496, + "active_bytes_peak": 73296223232, + "active_bytes_peak_delta": 32572506112, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 3, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.851738Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 31398314496, + "allocated_bytes_peak": 73296223232, + "allocated_bytes_peak_delta": 32572506112, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 3, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:05:59.393951Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_current_delta": 1967373824, + "inactive_split_bytes_peak": 4366457344, + "inactive_split_bytes_peak_delta": 3637868032, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 84802535424, + "reserved_bytes_current_delta": 40938504192, + "reserved_bytes_peak": 84802535424, + "reserved_bytes_peak_delta": 40938504192, + "segment_count_current": 1061, + "segment_count_current_delta": 800, + "segment_count_peak": 1061, + "segment_count_peak_delta": 800, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190638852254Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.888617Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190638852254Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190638852254Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4a3c162fcece55994b46b35c1bff51956d26808 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190638852254Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29a677982a440bce52d8168fa130f92d654f37709fa10eab0348ad60bff3d713 +size 2057276 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190639459374Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190639459374Z.json new file mode 100644 index 0000000000000000000000000000000000000000..62f10e88b89d2b97e812a155b663d8cad4f53792 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190639459374Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 21094154240, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 3, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4452327936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99042197504, + "reserved_bytes_peak": 99042197504, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:06:39.458934Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 21094154240, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 3, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.892980Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4452327936, + "inactive_split_bytes_peak_delta": 85870592, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99042197504, + "reserved_bytes_current_delta": 14239662080, + "reserved_bytes_peak": 99042197504, + "reserved_bytes_peak_delta": 14239662080, + "segment_count_current": 1132, + "segment_count_current_delta": 71, + "segment_count_peak": 1132, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190639459374Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.492096Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190639459374Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190639459374Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d6cabfe54fffa2f2436ffdfb27e0527ff1c9403 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190639459374Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ef4d4a4ec41efd9cfc1ca6d2d84ed5b10c3dcf12647d48e1c0f0448d3f7cf71 +size 1697214 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190639737411Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190639737411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..271452953545276ba4aef8b704040792617d0162 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190639737411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 3, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4452327936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.736960Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 3, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4452327936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 99042197504, + "reserved_bytes_peak": 99042197504, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:06:39.495913Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4452327936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99629400064, + "reserved_bytes_current_delta": 587202560, + "reserved_bytes_peak": 99629400064, + "reserved_bytes_peak_delta": 587202560, + "segment_count_current": 1134, + "segment_count_current_delta": 2, + "segment_count_peak": 1134, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190639737411Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.765104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190639737411Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190639737411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cadfcc13e92b880807031124e1b071c4b6d239de --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190639737411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea782eddb876f6e8f64f441380437e45ddf9c6a96b302b50d6ffaf463c8a53f +size 1689340 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190641564649Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190641564649Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3cd99f0d2932af9f46f3dc3780c9d767902ba71 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190641564649Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 4404695040, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 3, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.564211Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 675840, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 3, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4452327936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.768486Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": -138678272, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 2431932416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 8545894400, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 8545894400, + "segment_count_current": 1214, + "segment_count_current_delta": 80, + "segment_count_peak": 1214, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190641564649Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.600102Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190641564649Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190641564649Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0b48e90c08ee9f39d2f70b6edf2bcca86f3ee68 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190641564649Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff99a4d69581cce59b174fb9f6556d7c77917a3f074666cd6849f420a54ab4fa +size 2288101 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190641791336Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190641791336Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c87012baf9404ca0a68440cc79e11e47a96ed772 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190641791336Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 3, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.790937Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 3, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.604097Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190641791336Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798975Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190641791336Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190641791336Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7ee3919b4e0be594cf9965167fb232b54a674ea --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190641791336Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605511bd1cb69d2f36275bd488e8f507363512ff3f791e079a6d03d422964286 +size 364275 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190641979711Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190641979711Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6973b17d65f02081c352d736f2e4d7ea730b0d31 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190641979711Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 3, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.979320Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 3, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.799971Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190641979711Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190641979711Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190641979711Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ea554337d748962b00f4210ef9ccd77564d7f12 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190641979711Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27e1da163a765a8455bbc422f153893e72f422a6f92338d1b033a52f6c19be03 +size 364306 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190638851982Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190638851982Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1cef6104d95fadd1435365bcd82529ced492b091 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190638851982Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 31398314496, + "active_bytes_peak": 73296223232, + "active_bytes_peak_delta": 32572506112, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 4, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.851450Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 31398314496, + "allocated_bytes_peak": 73296223232, + "allocated_bytes_peak_delta": 32572506112, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 4, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:05:59.393854Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_current_delta": 1967373824, + "inactive_split_bytes_peak": 4366457344, + "inactive_split_bytes_peak_delta": 3637868032, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 84802535424, + "reserved_bytes_current_delta": 40938504192, + "reserved_bytes_peak": 84802535424, + "reserved_bytes_peak_delta": 40938504192, + "segment_count_current": 1061, + "segment_count_current_delta": 800, + "segment_count_peak": 1061, + "segment_count_peak_delta": 800, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190638851982Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.888008Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190638851982Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190638851982Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a58e64612c168e4d75ddf1ea983137fc1d67cd2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190638851982Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7757e1f5305f2263d3c8b0b316188a6d7488eccbabbc8ead12062411047be2a +size 2057276 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190639459340Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190639459340Z.json new file mode 100644 index 0000000000000000000000000000000000000000..26a57d053a2afe2994e3522b4cdc1b3d81efe3bd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190639459340Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 21094154240, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 4, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4412865024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.458903Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 21094154240, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 4, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.892135Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4412865024, + "inactive_split_bytes_peak_delta": 46407680, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99629400064, + "reserved_bytes_current_delta": 14826864640, + "reserved_bytes_peak": 99629400064, + "reserved_bytes_peak_delta": 14826864640, + "segment_count_current": 1134, + "segment_count_current_delta": 73, + "segment_count_peak": 1134, + "segment_count_peak_delta": 73, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190639459340Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.491826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190639459340Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190639459340Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0fd75e53cf36616a275580c38d70867e0f35c74b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190639459340Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d77b7339be8d1338bf2dd3f6a42793ff77d3a01c22ce28d7a4152fc9c192237 +size 1697522 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190639737360Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190639737360Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2fedf76f9837ba1ae1a123c90b56d44885469dd4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190639737360Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 4, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4419114496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.736958Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 4, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4412865024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.495541Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4419114496, + "inactive_split_bytes_peak_delta": 6249472, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99629400064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99629400064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1134, + "segment_count_current_delta": 0, + "segment_count_peak": 1134, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190639737360Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.764948Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190639737360Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190639737360Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51a6d25a84b65f95f3ddb5186d26f56db444f53b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190639737360Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5f44a6a30f9baa87096a914230de77f2520b1bd52cf6eb2d44ad50fde746b3b +size 1689215 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190641564646Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190641564646Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53542474739c2de6f5cdf5e24b6e136c3955e75b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190641564646Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 4404695040, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 4, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.564208Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 675840, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 4, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4419114496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.768246Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": -138678272, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 2465145856, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 8545894400, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 8545894400, + "segment_count_current": 1214, + "segment_count_current_delta": 80, + "segment_count_peak": 1214, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190641564646Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.598607Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190641564646Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190641564646Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d9a61dfeeefbe1ad2b9e6bdde7945d5a39d97d2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190641564646Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09490484beda67c13a1aa633b13c5e39179d6b188fcb0401b5176c01d74d216b +size 2091602 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190641791260Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190641791260Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13dd537f82027796d3ec7ec1475c303b9208d8ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190641791260Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 4, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.790857Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 4, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.602370Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190641791260Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798273Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190641791260Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190641791260Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4a065d7a980d4a564acb7927c3bbbccae565e49 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190641791260Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dad197de9b115a75272a984bc19220af1dacdd00d6183422d489fff1ee93d07 +size 336554 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190641979709Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190641979709Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c7b45d22a6d86682533a8175f49d3b6083224da --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190641979709Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 4, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.979322Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 4, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.799309Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190641979709Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986547Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190641979709Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190641979709Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98da9a67badfaea85e8fd18d5cc8430d47f0846a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190641979709Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:168a76de9a471a351df04882037d0e45e9ac029cada47f9b72701cd60b337f8b +size 336585 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190638851999Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190638851999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b5f32cc9fee60e4345cf7950d7e2eea5028e7695 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190638851999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 31398314496, + "active_bytes_peak": 73296223232, + "active_bytes_peak_delta": 32572506112, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 5, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 85096136704, + "reserved_bytes_peak": 85096136704, + "segment_count_current": 1062, + "segment_count_peak": 1062, + "timestamp_utc": "2026-08-26T19:06:38.851513Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 31398314496, + "allocated_bytes_peak": 73296223232, + "allocated_bytes_peak_delta": 32572506112, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 5, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:05:59.393836Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_current_delta": 1967373824, + "inactive_split_bytes_peak": 4366457344, + "inactive_split_bytes_peak_delta": 3637868032, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 85096136704, + "reserved_bytes_current_delta": 41232105472, + "reserved_bytes_peak": 85096136704, + "reserved_bytes_peak_delta": 41232105472, + "segment_count_current": 1062, + "segment_count_current_delta": 801, + "segment_count_peak": 1062, + "segment_count_peak_delta": 801, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190638851999Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.887694Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190638851999Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190638851999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3d58ef52ff497fbd699eb59b42a47e3c9d9a34c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190638851999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20585f06998e3a7b42907b167f0417a147341d7bfefd98f93faf35afea68744a +size 2057445 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190639459383Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190639459383Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f808dba6d81e7eba4c14526e738b47225d920f3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190639459383Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 21094154240, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 5, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4605152768, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.458946Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 21094154240, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 5, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 85096136704, + "reserved_bytes_peak": 85096136704, + "segment_count_current": 1062, + "segment_count_peak": 1062, + "timestamp_utc": "2026-08-26T19:06:38.891997Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4605152768, + "inactive_split_bytes_peak_delta": 238695424, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99923001344, + "reserved_bytes_current_delta": 14826864640, + "reserved_bytes_peak": 99923001344, + "reserved_bytes_peak_delta": 14826864640, + "segment_count_current": 1135, + "segment_count_current_delta": 73, + "segment_count_peak": 1135, + "segment_count_peak_delta": 73, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190639459383Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.492467Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190639459383Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190639459383Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f2fb0f03867f144b8f756bb7b811a8570dc5dfee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190639459383Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d7669c996670ee2dd147f83a11b224c6a560f0e5c89f96d40cef32a607d654 +size 1697629 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190639737384Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190639737384Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76a77f10949f4b8a0697d4b04b6aea938a31b0b9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190639737384Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 5, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4605152768, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.736987Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 5, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4605152768, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.496452Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4605152768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99923001344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99923001344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1135, + "segment_count_current_delta": 0, + "segment_count_peak": 1135, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190639737384Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.765969Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190639737384Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190639737384Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a61711db5e590ea06afcb4de1abb0d5695adc3f8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190639737384Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9fc936743751e0c4f455fbde6ee551e097d18793135076740780cc5d4969156 +size 1689337 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190641564688Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190641564688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bcadf8883cb1343c89b15648b6e41b1b007d1e9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190641564688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 4404695040, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 5, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.564246Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 675840, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 5, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4605152768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.769477Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": -138678272, + "inactive_split_bytes_peak": 7119141376, + "inactive_split_bytes_peak_delta": 2513988608, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 8252293120, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 8252293120, + "segment_count_current": 1214, + "segment_count_current_delta": 79, + "segment_count_peak": 1214, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190641564688Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.599364Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190641564688Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190641564688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c07e6a502774df40fddfb563c998220e354fb15 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190641564688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55efcbced93a20d979ba23b5d3062f6524f171cc8ea7047023303aa846559be3 +size 1974854 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190641791316Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190641791316Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3566fadf296fbd4f22036912c0023908d5df64a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190641791316Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 5, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.790907Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 5, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.603370Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7119141376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190641791316Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798505Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190641791316Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190641791316Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb7e33c22ebec4034c18600f3cf34957f3ac9fcc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190641791316Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d4101c9d0386a19e9946cb1c22a1f8736eade3dc8ed03690c8552f9d56bb848 +size 319569 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190641979792Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190641979792Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e9f2bdd26c3c79a2e3b6059774aeb259f598877 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190641979792Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 5, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.979391Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 5, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.799515Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7119141376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190641979792Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190641979792Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190641979792Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80e2f4669b211c97fd671edc0e94fcdf057b2fb5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190641979792Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ea96f9b7dc67b767c95c7e0b5d1fcfd1c73f27863fa698c27c3afa2d71ce0fc +size 319600 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190638852003Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190638852003Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f76fabf22eece163bb3e00f3f3fe2ee4ff3512bc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190638852003Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 31398314496, + "active_bytes_peak": 73296223232, + "active_bytes_peak_delta": 32572506112, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 6, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.851489Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 31398314496, + "allocated_bytes_peak": 73296223232, + "allocated_bytes_peak_delta": 32572506112, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 6, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:05:59.393856Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_current_delta": 1967373824, + "inactive_split_bytes_peak": 4366457344, + "inactive_split_bytes_peak_delta": 3637868032, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 84802535424, + "reserved_bytes_current_delta": 40938504192, + "reserved_bytes_peak": 84802535424, + "reserved_bytes_peak_delta": 40938504192, + "segment_count_current": 1061, + "segment_count_current_delta": 800, + "segment_count_peak": 1061, + "segment_count_peak_delta": 800, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190638852003Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.887083Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190638852003Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190638852003Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef921fac939e44ac10587b7af210a64137e401a5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190638852003Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c90397f0f91076ae377d1529b81a632e61360b0dddca65dc8375c3db4ee4a0fb +size 2057279 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190639459341Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190639459341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c8013e0aa66565cb53b3f1d5aac43670eb127c4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190639459341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 21094154240, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 6, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4729534976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.458910Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 21094154240, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 6, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.891289Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4729534976, + "inactive_split_bytes_peak_delta": 363077632, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 99923001344, + "reserved_bytes_current_delta": 15120465920, + "reserved_bytes_peak": 99923001344, + "reserved_bytes_peak_delta": 15120465920, + "segment_count_current": 1135, + "segment_count_current_delta": 74, + "segment_count_peak": 1135, + "segment_count_peak_delta": 74, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190639459341Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.491918Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190639459341Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190639459341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d55aeb41911b729ba139f22597a1a2f038248462 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190639459341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bdd474eeff80ac4529e865b5e4be124447e3a4cdca69aadb89ea9b977dec2cb +size 1697692 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190639737386Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190639737386Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c35d6d7e1dc74b7bcb6a326cd621deed0dd55fa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190639737386Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 6, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4729534976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.736965Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 6, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4729534976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.495712Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4729534976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 99923001344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99923001344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1135, + "segment_count_current_delta": 0, + "segment_count_peak": 1135, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190639737386Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.765135Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190639737386Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190639737386Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c379b1f374c14954d9397d95fb4b8a10007b9ed --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190639737386Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:623bc8c949617c42a1331e1dcf32a8d7e0d07b2560666d94e5b6393084418521 +size 1689400 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190641564647Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190641564647Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f698e197fc27aa2b49f1cb67a5e618b1647d4bf6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190641564647Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 4404695040, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 6, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.564217Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 675840, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 6, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4729534976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 99923001344, + "reserved_bytes_peak": 99923001344, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:06:39.768548Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": -138678272, + "inactive_split_bytes_peak": 7119141376, + "inactive_split_bytes_peak_delta": 2389606400, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 8252293120, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 8252293120, + "segment_count_current": 1214, + "segment_count_current_delta": 79, + "segment_count_peak": 1214, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190641564647Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.602530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190641564647Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190641564647Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..421ca8215e4d657be69d972b0a95acc8d3541261 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190641564647Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:489361d4740d5e6a776351116edd3192b0e4ed30b5a1ca6874c11d33117aecb8 +size 1999361 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190641791285Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190641791285Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac93399f927c9932f14046eecaa4e10a96222f4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190641791285Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 6, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.790876Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 6, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.606450Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7119141376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190641791285Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190641791285Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190641791285Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..18509578e933e863299b3190dc10cbc1f17f90c5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190641791285Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6448d93969308ef7753b54f79850b6aaf6833bde57036b078c16b026f2f2d054 +size 323124 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190641979712Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190641979712Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86ffb750e9cd0e29d250fb14cf0cc52138f37dd1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190641979712Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 6, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.979333Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 6, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 7119141376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.799343Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7119141376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190641979712Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986580Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190641979712Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190641979712Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d1535fe2697bb186ed3b6e92afc866db3cceb7eb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190641979712Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81472a4698db1f932dc4293d04cc7c7e79b2824237cb9491fd2aa432c4bd9e58 +size 323155 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190638852005Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190638852005Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c4dfe67d364b23109280982ad1d8717961d8be9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190638852005Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 31398314496, + "active_bytes_peak": 73296223232, + "active_bytes_peak_delta": 32572506112, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 7, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.851447Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 31398314496, + "allocated_bytes_peak": 73296223232, + "allocated_bytes_peak_delta": 32572506112, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 7, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:05:59.393815Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_current_delta": 1967373824, + "inactive_split_bytes_peak": 4366457344, + "inactive_split_bytes_peak_delta": 3637868032, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 84802535424, + "reserved_bytes_current_delta": 40938504192, + "reserved_bytes_peak": 84802535424, + "reserved_bytes_peak_delta": 40938504192, + "segment_count_current": 1061, + "segment_count_current_delta": 800, + "segment_count_peak": 1061, + "segment_count_peak_delta": 800, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190638852005Z.pickle", + "timestamp_utc": "2026-08-26T19:06:38.887033Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190638852005Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190638852005Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f7afb663039ee9ad3ab3826675e5edc1b98dd07 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190638852005Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54c86cb8ecb1b335b40b6bae70fec628e7bc6354f455c0205f4df75cc0eed140 +size 2057276 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190639459395Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190639459395Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed5de6658ed9f54665cdf8574144b2b231d1da9d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190639459395Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 21094154240, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 7, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4366457344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 99042197504, + "reserved_bytes_peak": 99042197504, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:06:39.458966Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 21094154240, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 73296223232, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 73296223232, + "device": 7, + "inactive_split_bytes_current": 2475762176, + "inactive_split_bytes_peak": 4366457344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 84802535424, + "reserved_bytes_peak": 84802535424, + "segment_count_current": 1061, + "segment_count_peak": 1061, + "timestamp_utc": "2026-08-26T19:06:38.891221Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4366457344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 99042197504, + "reserved_bytes_current_delta": 14239662080, + "reserved_bytes_peak": 99042197504, + "reserved_bytes_peak_delta": 14239662080, + "segment_count_current": 1132, + "segment_count_current_delta": 71, + "segment_count_peak": 1132, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190639459395Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.492105Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190639459395Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190639459395Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1087446277baf6cf15e340b5f3b07e1d19d821b5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190639459395Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d8360d1a801084de59ac1bbf5eae6a5a8fb70757e9a126e1d44e759a2759668 +size 1697178 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190639737413Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190639737413Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb8cda75e2f7dfd72c1d41763ddfbc2857589736 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190639737413Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084282880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94390377472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 7, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4498989568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.736987Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084282880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94390377472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 7, + "inactive_split_bytes_current": 2771460608, + "inactive_split_bytes_peak": 4366457344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 99042197504, + "reserved_bytes_peak": 99042197504, + "segment_count_current": 1132, + "segment_count_peak": 1132, + "timestamp_utc": "2026-08-26T19:06:39.495982Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4498989568, + "inactive_split_bytes_peak_delta": 132532224, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 99629400064, + "reserved_bytes_current_delta": 587202560, + "reserved_bytes_peak": 99629400064, + "reserved_bytes_peak_delta": 587202560, + "segment_count_current": 1134, + "segment_count_current_delta": 2, + "segment_count_peak": 1134, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190639737413Z.pickle", + "timestamp_utc": "2026-08-26T19:06:39.765327Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190639737413Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190639737413Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6eeea7e06b49b0e20bb131375ca0e3fdd4a2ea3d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190639737413Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d399d860c203b880067af614a62cdc0efb8328e2778998d24bc2b47f1ef8749 +size 1689313 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190641564687Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190641564687Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8daf925f769e6a7be833efd2bc47904d08656cb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190641564687Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 4404695040, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 7, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.564258Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 675840, + "before": { + "active_bytes_current": 72084282880, + "active_bytes_peak": 94390377472, + "allocated_bytes_current": 72084282880, + "allocated_bytes_peak": 94390377472, + "device": 7, + "inactive_split_bytes_current": 2769363456, + "inactive_split_bytes_peak": 4498989568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:06:39.768764Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": -138678272, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 2385270784, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 8545894400, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 8545894400, + "segment_count_current": 1214, + "segment_count_current_delta": 80, + "segment_count_peak": 1214, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190641564687Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.599486Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190641564687Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190641564687Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afc434dea280d86fd2b0f30ab729fa0042b7d5c8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190641564687Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a55768c3e7fc34c93c8fb980f9906fb39cff50029f24e0669aa67791387b24 +size 2288099 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190641791330Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190641791330Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ef4772a81ccee9c617cc0b0f1138fb770f7e5ea --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190641791330Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 7, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.790914Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 7, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.603342Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190641791330Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.798365Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190641791330Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190641791330Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..068b0bebe4fd276cc15c8d2a4aa1c3e9b18efd6c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190641791330Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0320d7bc28383918c959ca5b2b83f9090b34916722aec4c55fc9fb153ef72b92 +size 364273 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190641979742Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190641979742Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ef0918841152239043802e3f94dd9de811a5d39 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190641979742Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72101326336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 98795072512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 7, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.979362Z", + "world_size": 8 + }, + "allocated_bytes_current": 72101326336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94391053312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72101326336, + "active_bytes_peak": 98795072512, + "allocated_bytes_current": 72101326336, + "allocated_bytes_peak": 94391053312, + "device": 7, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_peak": 6884260352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108175294464, + "reserved_bytes_peak": 108175294464, + "segment_count_current": 1214, + "segment_count_peak": 1214, + "timestamp_utc": "2026-08-26T19:06:41.799440Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2630685184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6884260352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108175294464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108175294464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1214, + "segment_count_current_delta": 0, + "segment_count_peak": 1214, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190641979742Z.pickle", + "timestamp_utc": "2026-08-26T19:06:41.986674Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190641979742Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190641979742Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dba7678a239cebaa32212ed6764e80844ee7e6de --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190641979742Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a28a21fa7d05936f26a9fac0b86004301fe7b75f8619eea10048a85c58bb041 +size 364304 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..8d112ee1566bbbd807ff09a02b430ac03ca8b978 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ecfad842fc68c635177e817b03e0d4a41825a182 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -0,0 +1,1197 @@ +============================================================== + QWEN2_57B / echo (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.00 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 19:05:35.141000 610233 torch/distributed/run.py:766] +W0826 19:05:35.141000 610233 torch/distributed/run.py:766] ***************************************** +W0826 19:05:35.141000 610233 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:05:35.141000 610233 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W826 19:05:52.011055622 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:05:52.011067355 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:05:52.011068539 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:05:52.014784050 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:05:52.014838883 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:05:52.015017407 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:05:52.018580148 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:05:53.176287304 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.801 seconds +time to initialize megatron (seconds): 6.784 +[after megatron is initialized] datetime: 2026-08-26 19:05:58 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.bias +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.mlp.router.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.embedding.word_embeddings.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:05:59 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:05:59 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (334.83, 342.63) + train/valid/test-data-iterators-setup ..........: (61.75, 70.98) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:05:59 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:06:38] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 39456.8 | throughput per GPU (TFLOP/s/GPU): 3.1 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097923E+01 | loss scale: 1.0 | grad norm: 29.865 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 68744.94775390625 | max allocated: 69902.4853515625 | reserved: 81278.0 | max reserved: 81278.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:06:39] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 608.2 | throughput per GPU (TFLOP/s/GPU): 203.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097889E+01 | loss scale: 1.0 | grad norm: 30.618 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:06:39] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 278.0 | throughput per GPU (TFLOP/s/GPU): 444.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.539161E+00 | loss scale: 1.0 | grad norm: 32.431 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:06:41] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 1827.1 | throughput per GPU (TFLOP/s/GPU): 67.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.108423E+01 | loss scale: 1.0 | grad norm: 106.928 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:06:41] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 226.6 | throughput per GPU (TFLOP/s/GPU): 544.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.000449E+01 | loss scale: 1.0 | grad norm: 26.651 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:06:41] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 188.3 | throughput per GPU (TFLOP/s/GPU): 655.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.050018E+01 | loss scale: 1.0 | grad norm: 8.734 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:06:41 +[rank0]:[W826 19:06:43.122034776 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:06:43.822000 610233 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:06:43.823000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610322 closing signal SIGTERM +W0826 19:06:43.824000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610323 closing signal SIGTERM +W0826 19:06:43.826000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610324 closing signal SIGTERM +W0826 19:06:43.830000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610325 closing signal SIGTERM +W0826 19:06:43.833000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610326 closing signal SIGTERM +W0826 19:06:43.835000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610327 closing signal SIGTERM +W0826 19:06:43.838000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610328 closing signal SIGTERM +W0826 19:06:43.839000 610233 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 610329 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 610233 got signal: 15 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..92fdc8a3c561e3bbab8fd85553174666e8017191 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 27 +3, 629 +4, 629 +5, 629 +6, 35 +7, 27 +0, 967 +1, 981 +2, 987 +3, 979 +4, 977 +5, 987 +6, 989 +7, 995 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 43675 +1, 43135 +2, 43109 +3, 43821 +4, 43793 +5, 43829 +6, 43867 +7, 43825 +0, 44453 +1, 43915 +2, 43905 +3, 44627 +4, 44585 +5, 44575 +6, 44625 +7, 44575 +0, 47239 +1, 46671 +2, 46671 +3, 47371 +4, 47371 +5, 47371 +6, 47371 +7, 47371 +0, 47519 +1, 46951 +2, 46951 +3, 47651 +4, 47651 +5, 47651 +6, 47651 +7, 47651 +0, 47725 +1, 47159 +2, 47167 +3, 47871 +4, 47871 +5, 47877 +6, 47879 +7, 47889 +0, 48553 +1, 47985 +2, 47985 +3, 48685 +4, 48685 +5, 48685 +6, 48685 +7, 48685 +0, 48805 +1, 48237 +2, 48237 +3, 48937 +4, 48937 +5, 48937 +6, 48937 +7, 48937 +0, 49437 +1, 48867 +2, 48867 +3, 49567 +4, 49567 +5, 49567 +6, 49567 +7, 49567 +0, 50003 +1, 49435 +2, 49435 +3, 50135 +4, 50135 +5, 50135 +6, 50135 +7, 50135 +0, 77677 +1, 77649 +2, 77633 +3, 77657 +4, 77657 +5, 77657 +6, 77657 +7, 77657 +0, 78323 +1, 78295 +2, 78279 +3, 78303 +4, 78303 +5, 78303 +6, 78303 +7, 78303 +0, 78323 +1, 78295 +2, 78279 +3, 78303 +4, 78303 +5, 78303 +6, 78303 +7, 78303 +0, 78683 +1, 78655 +2, 78639 +3, 78663 +4, 78663 +5, 78663 +6, 78663 +7, 78663 +0, 78683 +1, 78655 +2, 78639 +3, 78663 +4, 78663 +5, 78663 +6, 78663 +7, 78663 +0, 78683 +1, 78655 +2, 78639 +3, 78663 +4, 78663 +5, 78663 +6, 78663 +7, 78663 +0, 79797 +1, 79769 +2, 79753 +3, 79777 +4, 79777 +5, 79777 +6, 79777 +7, 79777 +0, 81845 +1, 82937 +2, 81801 +3, 82665 +4, 81825 +5, 81825 +6, 81825 +7, 82105 +0, 89179 +1, 90377 +2, 89181 +3, 89787 +4, 88911 +5, 88979 +6, 88979 +7, 89217 +0, 104279 +1, 101027 +2, 103645 +3, 105507 +4, 112023 +5, 102697 +6, 101991 +7, 104083 +0, 112055 +1, 111917 +2, 111865 +3, 112023 +4, 111959 +5, 111761 +6, 112023 +7, 112023 +0, 2379 +1, 2638 +2, 110971 +3, 3662 +4, 2382 +5, 2382 +6, 2382 +7, 2382 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190753657106Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190753657106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3da60c8e9b66d682972187ff8bb337ab22dbab25 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190753657106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72080417280, + "active_bytes_current_delta": 31395495424, + "active_bytes_peak": 74043268096, + "active_bytes_peak_delta": 33320597504, + "after": { + "active_bytes_current": 72080417280, + "active_bytes_peak": 74043268096, + "allocated_bytes_current": 72080417280, + "allocated_bytes_peak": 74043268096, + "device": 0, + "inactive_split_bytes_current": 1865162240, + "inactive_split_bytes_peak": 4901234176, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 84464893952, + "reserved_bytes_peak": 84464893952, + "segment_count_current": 1034, + "segment_count_peak": 1034, + "timestamp_utc": "2026-08-26T19:07:53.656635Z", + "world_size": 8 + }, + "allocated_bytes_current": 72080417280, + "allocated_bytes_current_delta": 31395495424, + "allocated_bytes_peak": 74043268096, + "allocated_bytes_peak_delta": 33320597504, + "before": { + "active_bytes_current": 40684921856, + "active_bytes_peak": 40722670592, + "allocated_bytes_current": 40684921856, + "allocated_bytes_peak": 40722670592, + "device": 0, + "inactive_split_bytes_current": 373120000, + "inactive_split_bytes_peak": 593320960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 43723522048, + "reserved_bytes_peak": 43723522048, + "segment_count_current": 262, + "segment_count_peak": 262, + "timestamp_utc": "2026-08-26T19:07:14.578871Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1865162240, + "inactive_split_bytes_current_delta": 1492042240, + "inactive_split_bytes_peak": 4901234176, + "inactive_split_bytes_peak_delta": 4307913216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 84464893952, + "reserved_bytes_current_delta": 40741371904, + "reserved_bytes_peak": 84464893952, + "reserved_bytes_peak_delta": 40741371904, + "segment_count_current": 1034, + "segment_count_current_delta": 772, + "segment_count_peak": 1034, + "segment_count_peak_delta": 772, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190753657106Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.693901Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190753657106Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190753657106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..737d790261641b7bcf4b9c1c66a621763632516c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190753657106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6ec3c7c3e7f6dd2e74ea81b4b62b81e0a8dc56d5d5a5d2816ee38cc6a14adf4 +size 2061502 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190754288394Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190754288394Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9de119398c9e74cf7e3fdb7147370dca5bed303a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190754288394Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72080417280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95062694912, + "active_bytes_peak_delta": 21019426816, + "after": { + "active_bytes_current": 72080417280, + "active_bytes_peak": 95062694912, + "allocated_bytes_current": 72080417280, + "allocated_bytes_peak": 95062694912, + "device": 0, + "inactive_split_bytes_current": 2160860672, + "inactive_split_bytes_peak": 4901234176, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 100747182080, + "reserved_bytes_peak": 100747182080, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:07:54.287959Z", + "world_size": 8 + }, + "allocated_bytes_current": 72080417280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95062694912, + "allocated_bytes_peak_delta": 21019426816, + "before": { + "active_bytes_current": 72080417280, + "active_bytes_peak": 74043268096, + "allocated_bytes_current": 72080417280, + "allocated_bytes_peak": 74043268096, + "device": 0, + "inactive_split_bytes_current": 1865162240, + "inactive_split_bytes_peak": 4901234176, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 84464893952, + "reserved_bytes_peak": 84464893952, + "segment_count_current": 1034, + "segment_count_peak": 1034, + "timestamp_utc": "2026-08-26T19:07:53.698260Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2160860672, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4901234176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 100747182080, + "reserved_bytes_current_delta": 16282288128, + "reserved_bytes_peak": 100747182080, + "reserved_bytes_peak_delta": 16282288128, + "segment_count_current": 1130, + "segment_count_current_delta": 96, + "segment_count_peak": 1130, + "segment_count_peak_delta": 96, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190754288394Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.321314Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190754288394Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190754288394Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad74aef7ce0c3822c26ded73d5af0f4c47274dee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190754288394Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ff5b415909d78c4a83f6ae4fec97484254c8838f3aa9d8c93a50bcab39946fd +size 1711154 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190754565477Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190754565477Z.json new file mode 100644 index 0000000000000000000000000000000000000000..26adea0d904030037bd8fd0af94e504a87025a1f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190754565477Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72080417280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95062694912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72080417280, + "active_bytes_peak": 95062694912, + "allocated_bytes_current": 72080417280, + "allocated_bytes_peak": 95062694912, + "device": 0, + "inactive_split_bytes_current": 2160860672, + "inactive_split_bytes_peak": 4901234176, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 100747182080, + "reserved_bytes_peak": 100747182080, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:07:54.565067Z", + "world_size": 8 + }, + "allocated_bytes_current": 72080417280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95062694912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72080417280, + "active_bytes_peak": 95062694912, + "allocated_bytes_current": 72080417280, + "allocated_bytes_peak": 95062694912, + "device": 0, + "inactive_split_bytes_current": 2160860672, + "inactive_split_bytes_peak": 4901234176, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 100747182080, + "reserved_bytes_peak": 100747182080, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:07:54.325332Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2160860672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4901234176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 100747182080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 100747182080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1130, + "segment_count_current_delta": 0, + "segment_count_peak": 1130, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190754565477Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.594369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190754565477Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190754565477Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4905f69587fca2c14785d84d43b0afcc1d570996 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190754565477Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbb4e83972d8c5d15e7169cab1837d198f4349d98ecf58f6bde8804a29fc0bb6 +size 1725922 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190756150007Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190756150007Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a4089e7d6168417a08860ae275087ded825f977 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190756150007Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72097460736, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99476877312, + "active_bytes_peak_delta": 4414182400, + "after": { + "active_bytes_current": 72097460736, + "active_bytes_peak": 99476877312, + "allocated_bytes_current": 72097460736, + "allocated_bytes_peak": 95072858112, + "device": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_peak": 5738802688, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_peak": 108336775168, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.149559Z", + "world_size": 8 + }, + "allocated_bytes_current": 72097460736, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95072858112, + "allocated_bytes_peak_delta": 10163200, + "before": { + "active_bytes_current": 72080417280, + "active_bytes_peak": 95062694912, + "allocated_bytes_current": 72080417280, + "allocated_bytes_peak": 95062694912, + "device": 0, + "inactive_split_bytes_current": 2160860672, + "inactive_split_bytes_peak": 4901234176, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 100747182080, + "reserved_bytes_peak": 100747182080, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:07:54.597866Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_current_delta": -98832384, + "inactive_split_bytes_peak": 5738802688, + "inactive_split_bytes_peak_delta": 837568512, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_current_delta": 7589593088, + "reserved_bytes_peak": 108336775168, + "reserved_bytes_peak_delta": 7589593088, + "segment_count_current": 1166, + "segment_count_current_delta": 36, + "segment_count_peak": 1166, + "segment_count_peak_delta": 36, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190756150007Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.185373Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190756150007Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190756150007Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..942b3812df9409d652c19076b274bf411b3134f3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190756150007Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6972fce947cd863003e9b417fbf56eb9528369644db454bbcd13185655d1a8c7 +size 1938277 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190756378106Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190756378106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c84a718d6260d4b6b5dc119842d2c989527a37b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190756378106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72097460736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99476877312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72097460736, + "active_bytes_peak": 99476877312, + "allocated_bytes_current": 72097460736, + "allocated_bytes_peak": 95072858112, + "device": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_peak": 5738802688, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_peak": 108336775168, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.377686Z", + "world_size": 8 + }, + "allocated_bytes_current": 72097460736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95072858112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72097460736, + "active_bytes_peak": 99476877312, + "allocated_bytes_current": 72097460736, + "allocated_bytes_peak": 95072858112, + "device": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_peak": 5738802688, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_peak": 108336775168, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.189479Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5738802688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108336775168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1166, + "segment_count_current_delta": 0, + "segment_count_peak": 1166, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190756378106Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.385473Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190756378106Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190756378106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a3fd1e93c1a13d0f8d36c9ad29fc1c6ad2d6fa8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190756378106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aef539bbb8c1dc8c2e7a45194d1acca4a2786f3b27216bf2c85cef0e42ffa4a +size 312392 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190756568171Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190756568171Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98dc79300030b7ae29c2de44e96e74248d954c38 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190756568171Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72097460736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99476877312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72097460736, + "active_bytes_peak": 99476877312, + "allocated_bytes_current": 72097460736, + "allocated_bytes_peak": 95072858112, + "device": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_peak": 5738802688, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_peak": 108336775168, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.567785Z", + "world_size": 8 + }, + "allocated_bytes_current": 72097460736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95072858112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72097460736, + "active_bytes_peak": 99476877312, + "allocated_bytes_current": 72097460736, + "allocated_bytes_peak": 95072858112, + "device": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_peak": 5738802688, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_peak": 108336775168, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.386471Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2062028288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5738802688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108336775168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108336775168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1166, + "segment_count_current_delta": 0, + "segment_count_peak": 1166, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T190756568171Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.575216Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190756568171Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190756568171Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..10ff7eb8937ee4731dce812b2d0c5e4b7de03f4a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190756568171Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6414173a775d49adf1cfa36090b5d0d52b2e9679d36448632b62b119e3bec7d +size 312392 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190753657101Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190753657101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bcd3e2d09801ba5451d9d27f700201086ef30f6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190753657101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72068187136, + "active_bytes_current_delta": 31382218752, + "active_bytes_peak": 74051346432, + "active_bytes_peak_delta": 33327629312, + "after": { + "active_bytes_current": 72068187136, + "active_bytes_peak": 74051346432, + "allocated_bytes_current": 72068187136, + "allocated_bytes_peak": 74051346432, + "device": 1, + "inactive_split_bytes_current": 1736883200, + "inactive_split_bytes_peak": 4951808512, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 85723185152, + "reserved_bytes_peak": 85723185152, + "segment_count_current": 1052, + "segment_count_peak": 1052, + "timestamp_utc": "2026-08-26T19:07:53.656625Z", + "world_size": 8 + }, + "allocated_bytes_current": 72068187136, + "allocated_bytes_current_delta": 31382218752, + "allocated_bytes_peak": 74051346432, + "allocated_bytes_peak_delta": 33327629312, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 1, + "inactive_split_bytes_current": 302867456, + "inactive_split_bytes_peak": 523068416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:07:14.578824Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1736883200, + "inactive_split_bytes_current_delta": 1434015744, + "inactive_split_bytes_peak": 4951808512, + "inactive_split_bytes_peak_delta": 4428740096, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 85723185152, + "reserved_bytes_current_delta": 42593157120, + "reserved_bytes_peak": 85723185152, + "reserved_bytes_peak_delta": 42593157120, + "segment_count_current": 1052, + "segment_count_current_delta": 801, + "segment_count_peak": 1052, + "segment_count_peak_delta": 801, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190753657101Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.693086Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190753657101Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190753657101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c5b5d6e9416a25ab5c8e277031efd879ae28f7f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190753657101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6e546b5c9ec08b84075c7d095664f3e0102451b8bdea126243b1b290d00b73f +size 1800406 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190754288395Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190754288395Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b19dfdfb133b1bdab38c790d5bb41e3e5766ca58 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190754288395Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72068187136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95074207744, + "active_bytes_peak_delta": 21022861312, + "after": { + "active_bytes_current": 72068187136, + "active_bytes_peak": 95074207744, + "allocated_bytes_current": 72068187136, + "allocated_bytes_peak": 95074207744, + "device": 1, + "inactive_split_bytes_current": 2030484480, + "inactive_split_bytes_peak": 4951808512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 100717821952, + "reserved_bytes_peak": 100717821952, + "segment_count_current": 1144, + "segment_count_peak": 1144, + "timestamp_utc": "2026-08-26T19:07:54.287969Z", + "world_size": 8 + }, + "allocated_bytes_current": 72068187136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95074207744, + "allocated_bytes_peak_delta": 21022861312, + "before": { + "active_bytes_current": 72068187136, + "active_bytes_peak": 74051346432, + "allocated_bytes_current": 72068187136, + "allocated_bytes_peak": 74051346432, + "device": 1, + "inactive_split_bytes_current": 1736883200, + "inactive_split_bytes_peak": 4951808512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 85723185152, + "reserved_bytes_peak": 85723185152, + "segment_count_current": 1052, + "segment_count_peak": 1052, + "timestamp_utc": "2026-08-26T19:07:53.697571Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2030484480, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4951808512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 100717821952, + "reserved_bytes_current_delta": 14994636800, + "reserved_bytes_peak": 100717821952, + "reserved_bytes_peak_delta": 14994636800, + "segment_count_current": 1144, + "segment_count_current_delta": 92, + "segment_count_peak": 1144, + "segment_count_peak_delta": 92, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190754288395Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.321752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190754288395Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190754288395Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06ab2b75a7c64fd9216ab392d3106142837e1f2d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190754288395Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e735d8cbbf0dcd51ecb1f29d05ae21c2b698dd6c93555a88a57dac1825fd1530 +size 1675893 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..01f5e7105598d9867197e8e3f987320585452995 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72068187136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95074631168, + "active_bytes_peak_delta": 423424, + "after": { + "active_bytes_current": 72068187136, + "active_bytes_peak": 95074631168, + "allocated_bytes_current": 72068187136, + "allocated_bytes_peak": 95074631168, + "device": 1, + "inactive_split_bytes_current": 2028387328, + "inactive_split_bytes_peak": 4951808512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 100717821952, + "reserved_bytes_peak": 100717821952, + "segment_count_current": 1144, + "segment_count_peak": 1144, + "timestamp_utc": "2026-08-26T19:07:54.565047Z", + "world_size": 8 + }, + "allocated_bytes_current": 72068187136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95074631168, + "allocated_bytes_peak_delta": 423424, + "before": { + "active_bytes_current": 72068187136, + "active_bytes_peak": 95074207744, + "allocated_bytes_current": 72068187136, + "allocated_bytes_peak": 95074207744, + "device": 1, + "inactive_split_bytes_current": 2030484480, + "inactive_split_bytes_peak": 4951808512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 100717821952, + "reserved_bytes_peak": 100717821952, + "segment_count_current": 1144, + "segment_count_peak": 1144, + "timestamp_utc": "2026-08-26T19:07:54.325799Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2028387328, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4951808512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 100717821952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 100717821952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1144, + "segment_count_current_delta": 0, + "segment_count_peak": 1144, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.594414Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file